mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

Compare commits

base: mmr/tkDNN:v0.7
Branches Tags
mmr/tkDNN:master mmr/tkDNN:dev_rtls mmr/tkDNN:mbzirc mmr/tkDNN:tensorrt7 mmr/tkDNN:gpu-prepocess mmr/tkDNN:resize mmr/tkDNN:yolo4tiny mmr/tkDNN:cudnn-8.0.1 mmr/tkDNN:class mmr/tkDNN:cudnn5 mmr/tkDNN:cudnn2
mmr/tkDNN:v0.7 mmr/tkDNN:v0.6 mmr/tkDNN:v0.5 mmr/tkDNN:v0.4 mmr/tkDNN:testData
..
compare: mmr/tkDNN:resize
Branches Tags
mmr/tkDNN:master mmr/tkDNN:dev_rtls mmr/tkDNN:mbzirc mmr/tkDNN:tensorrt7 mmr/tkDNN:gpu-prepocess mmr/tkDNN:resize mmr/tkDNN:yolo4tiny mmr/tkDNN:cudnn-8.0.1 mmr/tkDNN:class mmr/tkDNN:cudnn5 mmr/tkDNN:cudnn2
mmr/tkDNN:v0.7 mmr/tkDNN:v0.6 mmr/tkDNN:v0.5 mmr/tkDNN:v0.4 mmr/tkDNN:testData

1 Commits
v0.7 .. resize

Author SHA1 Message Date
Micaela Verucchi 8ff7cad2e1 Modified resize and boxes coordinates to float, to achieve same darknet accuracy 
Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>
 2020-08-05 17:25:33 +02:00
177 changed files with 2104 additions and 25150 deletions
Expand all files Collapse all files
.gitignore
+1 -9
View File
@@ -12,13 +12,5 @@ build/
*.hdf5
*.pk
*.table
cmake-build-release/
demo/COCO_val2017
demo/BDD100K_val
/.vs
cmake-build-minsizerel/*
scripts/COCO_val2017/*
scripts/COCO_val2017.zip
scripts/all_labels.txt
/cmake/cuda_script
/cmake-build-debug/
demo/BDD100K_val
CMakeLists.txt
+15 -144
View File
@@ -1,69 +1,8 @@
cmake_minimum_required(VERSION 3.15)
project(tkDNN)
cmake_minimum_required(VERSION 3.5)
project (tkDNN)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
set(CMAKE_CXX_STANDARD 14)
option(ENABLE_OPENCV_CUDA_CONTRIB "Enable OpenCV CUDA Contrib" OFF )
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "default build" FORCE)
endif(NOT CMAKE_BUILD_TYPE)
find_package(CUDA 9.0 REQUIRED)
if (CUDA_FOUND)
set(OUTPUTFILE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cuda_script) # No suffix required
execute_process(COMMAND "rm ${OUTPUTFILE}")
set(CUDAFILE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/getCudaArch.cu)
execute_process(COMMAND ${CUDA_NVCC_EXECUTABLE} -lcuda ${CUDAFILE} -o ${OUTPUTFILE})
execute_process(COMMAND ${OUTPUTFILE}
RESULT_VARIABLE CUDA_RETURN_CODE
OUTPUT_VARIABLE ARCH)
if(${CUDA_RETURN_CODE} EQUAL 0)
set(CUDA_SUCCESS "TRUE")
else()
set(CUDA_SUCCESS "FALSE")
endif()
if (${CUDA_SUCCESS})
message(STATUS "CUDA Architecture: ${ARCH}")
message(STATUS "CUDA Version: ${CUDA_VERSION_STRING}")
message(STATUS "CUDA Path: ${CUDA_TOOLKIT_ROOT_DIR}")
message(STATUS "CUDA Libararies: ${CUDA_LIBRARIES}")
message(STATUS "CUDA Performance Primitives: ${CUDA_npp_LIBRARY}")
set(CUDA_NVCC_FLAGS "${ARCH}")
else()
message(WARNING ${ARCH})
endif()
endif()
SET(CUDA_SEPARABLE_COMPILATION ON)
if(UNIX)
if(CMAKE_BUILD_TYPE MATCHES Release)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wno-deprecated-declarations -Wno-unused-variable -O3")
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32)
endif()
if(CMAKE_BUILD_TYPE MATCHES Debug)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wno-deprecated-declarations -Wno-unused-variable -g3")
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32 -G -g)
endif()
endif()
if(WIN32)
if(CMAKE_BUILD_TYPE MATCHES Release)
set(CMAKE_CXX_FLAGS "/O2 /FS /EHsc /MD")
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32)
endif()
if(CMAKE_BUILD_TYPE MATCHES Debug)
set(CMAKE_CXX_FLAGS "/Od /FS /EHsc /MDd")
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32 -G -g)
endif()
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
endif(WIN32)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable")
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN)
# project specific flags
@@ -71,60 +10,37 @@ if(DEBUG)
add_definitions(-DDEBUG)
endif()
if(TKDNN_PATH)
message("SET TKDNN_PATH:" ${TKDNN_PATH})
add_definitions(-DTKDNN_PATH="${TKDNN_PATH}")
else()
add_definitions(-DTKDNN_PATH="${CMAKE_CURRENT_SOURCE_DIR}")
endif()
add_definitions(-DTKDNN_PATH="${CMAKE_CURRENT_SOURCE_DIR}")
#-------------------------------------------------------------------------------
# CUDA
#-------------------------------------------------------------------------------
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS}" --compiler-options '-fPIC')
find_package(CUDA 9.0 REQUIRED)
SET(CUDA_SEPARABLE_COMPILATION ON)
#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'")
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32)
find_package(CUDNN REQUIRED)
include_directories(${CUDNN_INCLUDE_DIR})
find_package(yaml-cpp REQUIRED)
# compile
file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu" "src/pluginsRT/*.cpp")
file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu")
cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS})
cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES} ${CUDA_LIBRARIES} ${CUDNN_LIBRARIES} yaml-cpp)
#-------------------------------------------------------------------------------
# External Libraries
#-------------------------------------------------------------------------------
find_package(Eigen3 REQUIRED)
message("Eigen DIR: " ${EIGEN3_INCLUDE_DIR})
include_directories(${EIGEN3_INCLUDE_DIR})
find_package(OpenCV REQUIRED)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV")
if(ENABLE_OPENCV_CUDA_CONTRIB)
if (OpenCV_FOUND)
find_package(OpenCV COMPONENTS cudawarping cudaarithm)
if(OpenCV_cudawarping_FOUND AND OpenCV_cudaarithm_FOUND)
add_compile_definitions(OPENCV_CUDACONTRIB)
message("OpenCV Cuda Contrib modules found")
else()
message("OpenCV Cuda Contrib modules not found")
set(ENABLE_OPENCV_CUDA_CONTRIB OFF)
endif()
endif()
endif()
# if(OpenCV_CUDA_VERSION)
# add_compile_definitions(OPENCV_CUDACONTRIB)
# endif()
# gives problems in cross-compiling, probably malformed cmake config
#find_package(yaml-cpp REQUIRED)
#-------------------------------------------------------------------------------
# Build Libraries
@@ -132,10 +48,10 @@ endif()
file(GLOB tkdnn_SRC "src/*.cpp")
set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS} yaml-cpp)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES})
add_library(tkDNN SHARED ${tkdnn_SRC})
target_link_libraries(tkDNN ${tkdnn_LIBS} ${CUDA_CUBLAS_LIBRARIES})
target_link_libraries(tkDNN ${tkdnn_LIBS})
#static
#add_library(tkDNN_static STATIC ${tkdnn_SRC})
@@ -161,7 +77,6 @@ foreach(test_SRC ${darknet_SRC})
set(test_NAME test_${test_NAME})
add_executable(${test_NAME} ${test_SRC})
target_link_libraries(${test_NAME} tkDNN)
install(TARGETS ${test_NAME} DESTINATION bin)
endforeach()
# MOBILENET
@@ -188,35 +103,6 @@ target_link_libraries(test_resnet101_cnet tkDNN)
add_executable(test_dla34_cnet tests/centernet/dla34_cnet/dla34_cnet.cpp)
target_link_libraries(test_dla34_cnet tkDNN)
add_executable(test_dla34_cnet3d tests/centernet/dla34_cnet3d/dla34_cnet3d.cpp)
target_link_libraries(test_dla34_cnet3d tkDNN)
# CENTERTRACK
add_executable(test_dla34_ctrack tests/centertrack/dla34_ctrack/dla34_ctrack.cpp)
target_link_libraries(test_dla34_ctrack tkDNN)
# SHELFNET
add_executable(test_shelfnet tests/shelfnet/shelfnet.cpp)
target_link_libraries(test_shelfnet tkDNN)
add_executable(test_shelfnet_berkeley tests/shelfnet/shelfnet_berkeley.cpp)
target_link_libraries(test_shelfnet_berkeley tkDNN)
add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp)
target_link_libraries(test_shelfnet_mapillary tkDNN)
add_executable(test_shelfnet_coco tests/shelfnet/shelfnet_coco.cpp)
target_link_libraries(test_shelfnet_coco tkDNN)
# MONODEPTH2
add_executable(test_monodepth2_640 tests/monodepth2/monodepth2_640.cpp)
target_link_libraries(test_monodepth2_640 tkDNN)
add_executable(test_monodepth2_1024 tests/monodepth2/monodepth2_1024.cpp)
target_link_libraries(test_monodepth2_1024 tkDNN)
# DEMOS
add_executable(test_rtinference tests/test_rtinference/rtinference.cpp)
target_link_libraries(test_rtinference tkDNN)
@@ -227,18 +113,6 @@ target_link_libraries(map_demo tkDNN)
add_executable(demo demo/demo/demo.cpp)
target_link_libraries(demo tkDNN)
add_executable(demo3D demo/demo/demo3D.cpp)
target_link_libraries(demo3D tkDNN)
add_executable(demoTracker demo/demo/demoTracker.cpp)
target_link_libraries(demoTracker tkDNN)
add_executable(seg_demo demo/demo/seg_demo.cpp)
target_link_libraries(seg_demo tkDNN)
add_executable(demoDepth demo/demo/demoDepth.cpp)
target_link_libraries(demoDepth tkDNN)
#-------------------------------------------------------------------------------
# Install
#-------------------------------------------------------------------------------
@@ -248,11 +122,8 @@ target_link_libraries(demoDepth tkDNN)
#endif()
message("install dir:" ${CMAKE_INSTALL_PREFIX})
install(DIRECTORY include/ DESTINATION include/)
install(TARGETS tkDNN DESTINATION lib)
install(TARGETS test_simple test_mnist test_mnistRT test_rtinference demo map_demo DESTINATION bin)
install(TARGETS tkDNN kernels DESTINATION lib)
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/cmake/" # source directory
DESTINATION "share/tkDNN/cmake/" # target directory
)
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tests/" # source directory
DESTINATION "share/tkDNN/tests" # target directory
)
README.md
+249 -120
View File
@@ -3,123 +3,88 @@ tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives,
The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training.
If you use tkDNN in your research, please cite the [following paper](https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9212130&casa_token=sQTJXi7tJNoAAAAA:BguH9xCIY48MxbtDS3LXzIXzO-9sWArm7Hd7y7BwaLmqRuM_Gx8bOYizFPNMNtpo5K0kB-P-). For use in commercial solutions, write at gattifrancesco@hotmail.it and micaela.verucchi@unimore.it or refer to https://hipert.unimore.it/ .
If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it and micaela.verucchi@unimore.it or refer to https://hipert.unimore.it/ .
```
@inproceedings{verucchi2020systematic,
title={A Systematic Assessment of Embedded Neural Networks for Object Detection},
author={Verucchi, Micaela and Brilli, Gianluca and Sapienza, Davide and Verasani, Mattia and Arena, Marco and Gatti, Francesco and Capotondi, Alessandro and Cavicchioli, Roberto and Bertogna, Marko and Solieri, Marco},
booktitle={2020 25th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)},
volume={1},
pages={937--944},
year={2020},
organization={IEEE}
}
Accepted paper @ IRC 2020, will soon be published.
M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Time clustering and LiDAR-camera fusion on embedded platforms for self-driving cars", in proceedings in IEEE Robotic Computing (2020)
Accepted paper @ ETFA 2020, will soon be published.
M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capotondi, R. Cavicchioli, M. Bertogna, M. Solieri
"A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020)
```
### What's new
#### 20 July 2021
- [x] Support to sematic segmentation [README](docs/README_seg.md)
- [x] Support 2D/3D Object Detection and Tracking [README](docs/README_2d3dtracking.md)
#### 24 November 2021
- [x] Support to sematic segmentation on cuda 11
- [x] Support to TensorRT8. (thanks to [Harshvardhan Chandirasekar](https://github.com/perseusdg))
#### 30 March 2022
- [x] Support to monocular depth esitmation [README](docs/README_depth.md) (thanks to [Harshvardhan Chandirasekar](https://github.com/perseusdg))
## FPS Results
Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on
## Results
Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesion as the input size, on
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
* Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
* Xavier NX, Jetpack 4.4 (CUDA 10.2, CUDNN 8.0.0, tensorrt 7.1.0 ).
* Tx2, Jetpack 4.2 (CUDA 10.0, CUDNN 7.3.1, tensorrt 5.0.6 );
* Jetson Nano, Jetpack 4.4 (CUDA 10.2, CUDNN 8.0.0, tensorrt 7.1.0 ).
| Platform | Network | FP32, B=1 | FP32, B=4 | FP16, B=1 | FP16, B=4 | INT8, B=1 | INT8, B=4 |
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
| RTX 2080Ti | yolo4 320 | 118.59 | 237.31 | 207.81 | 443.32 | 262.37 | 530.93 |
| RTX 2080Ti | yolo4 416 | 104.81 | 162.86 | 169.06 | 293.78 | 206.93 | 353.26 |
| RTX 2080Ti | yolo4 512 | 92.98 | 132.43 | 140.36 | 215.17 | 165.35 | 254.96 |
| RTX 2080Ti | yolo4 608 | 63.77 | 81.53 | 111.39 | 152.89 | 127.79 | 184.72 |
| AGX Xavier | yolo4 320 | 26.78 | 32.05 | 57.14 | 79.05 | 73.15 | 97.56 |
| AGX Xavier | yolo4 416 | 19.96 | 21.52 | 41.01 | 49.00 | 50.81 | 60.61 |
| AGX Xavier | yolo4 512 | 16.58 | 16.98 | 31.12 | 33.84 | 37.82 | 41.28 |
| AGX Xavier | yolo4 608 | 9.45 | 10.13 | 21.92 | 23.36 | 27.05 | 28.93 |
| Xavier NX | yolo4 320 | 14.56 | 16.25 | 30.14 | 41.15 | 42.13 | 53.42 |
| Xavier NX | yolo4 416 | 10.02 | 10.60 | 22.43 | 25.59 | 29.08 | 32.94 |
| Xavier NX | yolo4 512 | 8.10 | 8.32 | 15.78 | 17.13 | 20.51 | 22.46 |
| Xavier NX | yolo4 608 | 5.26 | 5.18 | 11.54 | 12.06 | 15.09 | 15.82 |
| Tx2 | yolo4 320 | 11.18 | 12.07 | 15.32 | 16.31 | - | - |
| Tx2 | yolo4 416 | 7.30 | 7.58 | 9.45 | 9.90 | - | - |
| Tx2 | yolo4 512 | 5.96 | 5.95 | 7.22 | 7.23 | - | - |
| Tx2 | yolo4 608 | 3.63 | 3.65 | 4.67 | 4.70 | - | - |
| Nano | yolo4 320 | 4.23 | 4.55 | 6.14 | 6.53 | - | - |
| Nano | yolo4 416 | 2.88 | 3.00 | 3.90 | 4.04 | - | - |
| Nano | yolo4 512 | 2.32 | 2.34 | 3.02 | 3.04 | - | - |
| Nano | yolo4 608 | 1.40 | 1.41 | 1.92 | 1.93 | - | - |
## MAP Results
Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001
| | CodaLab | CodaLab | CodaLab | CodaLab | tkDNN map | tkDNN map |
| -------------------- | :-----------: | :-------: | :-----------: | :---------: | :-----------: | :-------: |
| | **tkDNN** | **tkDNN** | **darknet** | **darknet** | **tkDNN** | **tkDNN** |
| | MAP(0.5:0.95) | AP50 | MAP(0.5:0.95) | AP50 | MAP(0.5:0.95) | AP50 |
| Yolov3 (416x416) | 0.381 | 0.675 | 0.380 | 0.675 | 0.372 | 0.663 |
| yolov4 (416x416) | 0.468 | 0.705 | 0.471 | 0.710 | 0.459 | 0.695 |
| yolov3tiny (416x416) | 0.096 | 0.202 | 0.096 | 0.201 | 0.093 | 0.198 |
| yolov4tiny (416x416) | 0.202 | 0.400 | 0.201 | 0.400 | 0.197 | 0.395 |
| Cnet-dla34 (512x512) | 0.366 | 0.543 | \- | \- | 0.361 | 0.535 |
| mv2SSD (512x512) | 0.226 | 0.381 | \- | \- | 0.223 | 0.378 |
| RTX 2080Ti | yolo4 320 | 118,59 |237,31 | 207,81 | 443,32 | 262,37 | 530,93 |
| RTX 2080Ti | yolo4 416 | 104,81 |162,86 | 169,06 | 293,78 | 206,93 | 353,26 |
| RTX 2080Ti | yolo4 512 | 92,98 |132,43 | 140,36 | 215,17 | 165,35 | 254,96 |
| RTX 2080Ti | yolo4 608 | 63,77 |81,53 | 111,39 | 152,89 | 127,79 | 184,72 |
| AGX Xavier | yolo4 320 | 26,78 |32,05 | 57,14 | 79,05 | 73,15 | 97,56 |
| AGX Xavier | yolo4 416 | 19,96 |21,52 | 41,01 | 49,00 | 50,81 | 60,61 |
| AGX Xavier | yolo4 512 | 16,58 |16,98 | 31,12 | 33,84 | 37,82 | 41,28 |
| AGX Xavier | yolo4 608 | 9,45 |10,13 | 21,92 | 23,36 | 27,05 | 28,93 |
| Tx2 | yolo4 320 | 11,18 | 12,07 | 15,32 | 16,31 | - | - |
| Tx2 | yolo4 416 | 7,30 | 7,58 | 9,45 | 9,90 | - | - |
| Tx2 | yolo4 512 | 5,96 | 5,95 | 7,22 | 7,23 | - | - |
| Tx2 | yolo4 608 | 3,63 | 3,65 | 4,67 | 4,70 | - | - |
| Nano | yolo4 320 | 4,23 | 4,55 | 6,14 | 6,53 | - | - |
| Nano | yolo4 416 | 2,88 | 3,00 | 3,90 | 4,04 | - | - |
| Nano | yolo4 512 | 2,32 | 2,34 | 3,02 | 3,04 | - | - |
| Nano | yolo4 608 | 1,40 | 1,41 | 1,92 | 1,93 | - | - |
## Index
- [tkDNN](#tkdnn)
- [Index](#index)
- [Dependencies](#dependencies)
- [About OpenCV](#about-opencv)
- [How to compile this repo](#how-to-compile-this-repo)
- [Workflow](#workflow)
- [Exporting weights](#exporting-weights)
- [Run the demos](#run-the-demos)
- [tkDNN on Windows 10 or Windows 11](#tkdnn-on-windows-10-or-windows-11)
- [How to export weights](#how-to-export-weights)
- [1)Export weights from darknet](#1export-weights-from-darknet)
- [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101)
- [3)Export weights for CenterNet](#3export-weights-for-centernet)
- [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd)
- [Run the demo](#run-the-demo)
- [FP16 inference](#fp16-inference)
- [INT8 inference](#int8-inference)
- [mAP demo](#map-demo)
- [Existing tests and supported networks](#existing-tests-and-supported-networks)
- [References](#references)
## Dependencies
This branch works on every NVIDIA GPU that supports the following (latest tested) dependencies:
* CUDA 11.3 (or >= 10.2)
* cuDNN 8.2.1 (or >= 8.0.4)
* TensorRT 8.0.3 (or >=7.2)
* OpenCV 4.5.4 (or >=4)
* cmake 3.21 (or >= 3.15)
* yaml-cpp 0.5.2
* eigen3 3.3.4
* curl 7.58
This branch works on every NVIDIA GPU that supports the dependencies:
* CUDA 10.0
* CUDNN 7.603
* TENSORRT 6.01
* OPENCV 3.4
* yaml-cpp 0.5.2 (sudo apt install libyaml-cpp-dev)
```
sudo apt install libyaml-cpp-dev curl libeigen3-dev
```
#### About OpenCV
## About OpenCV
To compile and install OpenCV4 with contrib us the script ```install_OpenCV4.sh```. It will download and compile OpenCV in Download folder.
```
bash scripts/install_OpenCV4.sh
```
If you have OpenCV compiled with cuda and contrib and want to use it with tkDNN pass ```ENABLE_OPENCV_CUDA_CONTRIB=ON``` flag when compiling tkDBB
. If the flag is not passed,the preprocessing of the networks is computed on the CPU, otherwise on the GPU. In the latter case some milliseconds are saved in the end-to-end latency.
When using openCV not compiled with contrib, comment the definition of OPENCV_CUDACONTRIBCONTRIB in include/tkDNN/DetectionNN.h. When commented, the preprocessing of the networks is computed on the CPU, otherwise on the GPU. In the latter case some milliseconds are saved in the end-to-end latency.
## How to compile this repo
Build with cmake. If using Ubuntu 18.04 a new version of cmake is needed (3.15 or above).
On both linux and windows ,the ```CMAKE_BUILD_TYPE``` variable needs to be defined as either ```Release``` or ```Debug```.
Build with cmake. If using Ubuntu 18.04 a new version of cmake is needed (3.15 or above).
```
git clone https://github.com/ceccocats/tkDNN
cd tkDNN
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
cmake ..
make
```
@@ -131,24 +96,212 @@ Steps needed to do inference on tkDNN with a custom neural network.
* Create a new test and define the network, layer by layer using the weights extracted and the output to check the results.
* Do inference.
## Exporting weights
## How to export weights
For specific details on how to export weights see [HERE](./docs/exporting_weights.md).
Weights are essential for any network to run inference. For each test a folder organized as follow is needed (in the build folder):
```
test_nn
|---- layers/ (folder containing a binary file for each layer with the corresponding wieghts and bias)
|---- debug/ (folder containing a binary file for each layer with the corresponding outputs)
```
Therefore, once the weights have been exported, the folders layers and debug should be placed in the corresponding test.
## Run the demos
### 1)Export weights from darknet
To export weights for NNs that are defined in darknet framework, use [this](https://git.hipert.unimore.it/fgatti/darknet.git) fork of darknet and follow these steps to obtain a correct debug and layers folder, ready for tkDNN.
```
git clone https://git.hipert.unimore.it/fgatti/darknet.git
cd darknet
make
mkdir layers debug
./darknet export <path-to-cfg-file> <path-to-weights> layers
```
N.b. Use compilation with CPU (leave GPU=0 in Makefile) if you also want debug.
### 2)Export weights for DLA34 and ResNet101
To get weights and outputs needed to run the tests dla34 and resnet101 use the Python script and the Anaconda environment included in the repository.
Create Anaconda environment and activate it:
```
conda env create -f file_name.yml
source activate env_name
python <script name>
```
### 3)Export weights for CenterNet
To get the weights needed to run Centernet tests use [this](https://github.com/sapienzadavide/CenterNet.git) fork of the original Centernet.
```
git clone https://github.com/sapienzadavide/CenterNet.git
```
* follow the instruction in the README.md and INSTALL.md
```
python demo.py --input_res 512 --arch resdcn_101 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_resdcn101.pth --exp_wo --exp_wo_dim 512
python demo.py --input_res 512 --arch dla_34 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_dla_2x.pth --exp_wo --exp_wo_dim 512
```
### 4)Export weights for MobileNetSSD
To get the weights needed to run Mobilenet tests use [this](https://github.com/mive93/pytorch-ssd) fork of a Pytorch implementation of SSD network.
```
git clone https://github.com/mive93/pytorch-ssd
cd pytorch-ssd
conda env create -f env_mobv2ssd.yml
python run_ssd_live_demo.py mb2-ssd-lite <pth-model-fil> <labels-file>
```
## Darknet Parser
tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*:
```
// example of parsing yolo4
tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names");
net->print();
```
All models from darknet are now parsed directly from cfg, you still need to export the weights with the descripted tools in the previus section.
<details>
<summary>Supported layers</summary>
convolutional
maxpool
avgpool
shortcut
upsample
route
reorg
region
yolo
</details>
<details>
<summary>Supported activations</summary>
relu
leaky
mish
</details>
## Run the demo
This is an example using yolov4.
To run the an object detection first create the .rt file by running:
```
rm yolo4_fp32.rt # be sure to delete(or move) old tensorRT files
./test_yolo4 # run the yolo test (is slow)
```
If you get problems in the creation, try to check the error activating the debug of TensorRT in this way:
```
cmake .. -DDEBUG=True
make
```
Once you have succesfully created your rt file, run the demo:
```
./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y
```
In general the demo program takes 6 parameters:
```
./demo <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes> <n-batches> <show-flag>
```
where
* ```<network-rt-file>``` is the rt file generated by a test
* ```<<path-to-video>``` is the path to a video file or a camera input
* ```<kind-of-network>``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
* ```<number-of-classes>```is the number of classes the network is trained on
* ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
* ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
N.b. By default it is used FP32 inference
For specific details on how to run:
- 2D object detection demos, details on FP16, INT8 and batching see [HERE](./docs/demo.md).
- segmentation demos see [HERE](./docs/README_seg.md).
- monocular depth estimation see [HERE](./docs/README_depth.md).
- 2D/3D object detection and tracking demos see [HERE](./docs/README_2d3dtracking.md).
- mAP demo to evaluate 2D object detectors see [HERE](./docs/mAP_demo.md).
![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif)
## tkDNN on Windows 10 or Windows 11
### FP16 inference
For specific details on how to run tkDNN on Windows 10/11 see [HERE](./docs/windows.md).
To run the an object detection demo with FP16 inference follow these steps (example with yolov3):
```
export TKDNN_MODE=FP16 # set the half floating point optimization
rm yolo3_fp16.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # run the yolo test (is slow)
./demo yolo3_fp16.rt ../demo/yolo_test.mp4 y
```
N.b. Using FP16 inference will lead to some errors in the results (first or second decimal).
### INT8 inference
To run the an object detection demo with INT8 inference three environment variables need to be set:
* ```export TKDNN_MODE=INT8```: set the 8-bit integer optimization
* ```export TKDNN_CALIB_IMG_PATH=/path/to/calibration/image_list.txt``` : image_list.txt has in each line the absolute path to a calibration image
* ```export TKDNN_CALIB_LABEL_PATH=/path/to/calibration/label_list.txt```: label_list.txt has in each line the absolute path to a calibration label
You should provide image_list.txt and label_list.txt, using training images. However, if you want to quickly test the INT8 inference you can run (from this repo root folder)
```
bash scripts/download_validation.sh COCO
```
to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD insted of COCO to download BDD validation.
Then a complete example using yolo3 and COCO dataset would be:
```
export TKDNN_MODE=INT8
export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
rm yolo3_int8.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # run the yolo test (is slow)
./demo yolo3_int8.rt ../demo/yolo_test.mp4 y
```
N.B.
* Using INT8 inference will lead to some errors in the results.
* The test will be slower: this is due to the INT8 calibration, which may take some time to complete.
* INT8 calibration requires TensorRT version greater than or equal to 6.0
* Only 100 images are used to create the calibration table by default (set in the code).
### BatchSize bigger than 1
```
export TKDNN_BATCHSIZE=2
# build tensorRT files
```
This will create a TensorRT file with the desidered **max** batch size.
The test will still run with a batch of 1, but the created tensorRT can manage the desidered batch size.
### Test batch Inference
This will test the network with random input and check if the output of each batch is the same.
```
./test_rtinference <network-rt-file> <number-of-batches>
# <number-of-batches> should be less or equal to the max batch size of the <network-rt-file>
# example
export TKDNN_BATCHSIZE=4 # set max batch size
rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # build RT file
./test_rtinference yolo3_fp32.rt 4 # test with a batch size of 4
```
## mAP demo
To compute mAP, precision, recall and f1score, run the map_demo.
A validation set is needed.
To download COCO_val2017 (80 classes) run (form the root folder):
```
bash scripts/download_validation.sh COCO
```
To download Berkeley_val (10 classes) run (form the root folder):
```
bash scripts/download_validation.sh BDD
```
To compute the map, the following parameters are needed:
```
./map_demo <network rt> <network type [y|c|m]> <labels file path> <config file path>
```
where
* ```<network rt>```: rt file of a chosen network on which compute the mAP.
* ```<network type [y|c|m]>```: type of network. Right now only y(yolo), c(centernet) and m(mobilenet) are allowed
* ```<labels file path>```: path to a text file containing all the paths of the ground-truth labels. It is important that all the labels of the ground-truth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the ground-truth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg.
* ```<config file path>```: path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml
Example:
```
cd build
./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml
```
This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to subit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate).
## Existing tests and supported networks
@@ -175,20 +328,8 @@ For specific details on how to run tkDNN on Windows 10/11 see [HERE](./docs/wind
| resnet101_cnet | Centernet (Resnet101 backend)<sup>4</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download) |
| csresnext50-panet-spp | Cross Stage Partial Network <sup>7</sup> | [COCO 2014](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download) |
| yolo4 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
| yolo4_320 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 320x320 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
| yolo4_512 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
| yolo4_608 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 608x608 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
| yolo4_berkeley | Yolov4 <sup>8</sup> | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 544x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) |
| yolo4tiny | Yolov4 tiny <sup>9</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) |
| yolo4x | Yolov4x-mish <sup>9</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 640x640 | [weights](https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download) |
| yolo4tiny_512 | Yolov4 tiny <sup>9</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) |
| yolo4x-cps | Scaled Yolov4 <sup>10</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/AfzHE4BfTeEm2gH/download) |
| shelfnet | ShelfNet18_realtime<sup>11</sup> | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) |
| shelfnet_berkeley | ShelfNet18_realtime<sup>11</sup> | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) |
| dla34_cnet3d | Centernet3D (DLA34 backend)<sup>4</sup> | [KITTI 2017](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d) | 1 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/2MDyWGzQsTKMjmR/download) |
| dla34_ctrack | CenterTrack (DLA34 backend)<sup>12</sup> | [NuScenes 3D](https://www.nuscenes.org/) | 7 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download) |
| monodepth2 | Monodepth2 <sup>13</sup> | [KITTI DEPTH](http://www.cvlibs.net/datasets/kitti/raw_data.php) | - | 640x192 | [weights-mono](https://cloud.hipert.unimore.it/s/iYw9QwgP6CsqxLR/download) |
| monodepth2 | Monodepth2 <sup>13</sup> | [KITTI DEPTH](http://www.cvlibs.net/datasets/kitti/raw_data.php) | - | 640x192 | [weights-stereo](https://cloud.hipert.unimore.it/s/XmwbWNXDfqyQ4EL/download) |
| yolo4_berkeley | Yolov4 <sup>8</sup> | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 540x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) |
| yolo4tiny | Yolov4 tiny | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) |
## References
@@ -201,15 +342,3 @@ For specific details on how to run tkDNN on Windows 10/11 see [HERE](./docs/wind
6. He, Kaiming, et al. "Deep residual learning for image recognition." Proceedings of the IEEE conference on computer vision and pattern recognition. 2016.
7. Wang, Chien-Yao, et al. "CSPNet: A New Backbone that can Enhance Learning Capability of CNN." arXiv preprint arXiv:1911.11929 (2019).
8. Bochkovskiy, Alexey, Chien-Yao Wang, and Hong-Yuan Mark Liao. "YOLOv4: Optimal Speed and Accuracy of Object Detection." arXiv preprint arXiv:2004.10934 (2020).
9. Bochkovskiy, Alexey, "Yolo v4, v3 and v2 for Windows and Linux" (https://github.com/AlexeyAB/darknet)
10. Wang, Chien-Yao, Alexey Bochkovskiy, and Hong-Yuan Mark Liao. "Scaled-YOLOv4: Scaling Cross Stage Partial Network." arXiv preprint arXiv:2011.08036 (2020).
11. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019.
12. Zhou, Xingyi, Vladlen Koltun, and Philipp Krähenbühl. "Tracking objects as points." European Conference on Computer Vision. Springer, Cham, 2020.
13. Godard, Clément, et al. "Digging into self-supervised monocular depth estimation." Proceedings of the IEEE/CVF International Conference on Computer Vision. 2019.
## Contributors
The main contibutors, in chronological order, are:
- [Francesco Gatti](https://github.com/ceccocats), francesco.gatti@hipert.it
- [Micaela Verucchi](https://github.com/mive93), micaela.verucchi@unimore.it
- [Davide Sapienza](https://github.com/sapienzadavide), davide.sapienza@unimore.it
- [Harshvardhan Chandirasekar](https://github.com/perseusdg), f20180523@goa.bits-pilani.ac.in
cmake/getCudaArch.cu
-20
View File
@@ -1,20 +0,0 @@
#include <stdio.h>
int main(int argc, char **argv){
cudaDeviceProp dP;
float min_cc = 5.0;
int rc = cudaGetDeviceProperties(&dP, 0);
if(rc != cudaSuccess) {
cudaError_t error = cudaGetLastError();
printf("CUDA error: %s", cudaGetErrorString(error));
return rc; /* Failure */
}
if((dP.major+(dP.minor/10)) < min_cc) {
printf("Min Compute Capability of %2.1f required: %d.%d found\n Not Building CUDA Code", min_cc, dP.major, dP.minor);
return 1; /* Failure */
} else {
printf("-arch=sm_%d%d", dP.major, dP.minor);
return 0; /* Success */
}
}
demo/config.yaml
+1 -1
View File
@@ -3,5 +3,5 @@ map_points : 101 #number of recall points (0 for all, 101 for COCO, 11 Pascal
map_levels : 10 #number of IoU step for the AP
map_step : 0.05 #step of IoU
IoU_thresh : 0.5 #starting IoU threshold
conf_thresh : 0.001 #threshold on the condifence of the bbox
conf_thresh : 0.0 #threshold on the condifence of the bbox
verbose : false #print on screen information
demo/demo/demo.cpp
+36 -53
View File
@@ -1,7 +1,7 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
//#include <unistd.h>
#include <unistd.h>
#include <mutex>
#include "CenternetDetection.h"
@@ -9,6 +9,7 @@
#include "Yolo3Detection.h"
bool gRun;
bool SAVE_RESULT = false;
void sig_handler(int signo) {
std::cout<<"request gateway stop\n";
@@ -17,56 +18,38 @@ void sig_handler(int signo) {
int main(int argc, char *argv[]) {
std::cout<<"detection\n";
signal(SIGINT, sig_handler);
// get config file path and read it
#ifdef __linux__
std::string config_file = "../demo/demoConfig.yaml";
#elif _WIN32
std::string config_file = "..\\..\\..\\demo\\demoConfig.yaml";
#endif
std::string net = "yolo3_berkeley.rt";
if(argc > 1)
config_file = argv[1];
YAML::Node conf = YAMLloadConf(config_file);
if(!conf)
FatalError("Problem with config file");
net = argv[1];
std::string input = "../demo/yolo_test.mp4";
if(argc > 2)
input = argv[2];
char ntype = 'y';
if(argc > 3)
ntype = argv[3][0];
int n_classes = 80;
if(argc > 4)
n_classes = atoi(argv[4]);
int n_batch = 1;
if(argc > 5)
n_batch = atoi(argv[5]);
bool show = true;
if(argc > 6)
show = atoi(argv[6]);
// read settings from config file
std::string net = YAMLgetConf<std::string>(conf, "net", "yolo4tiny_fp32.rt");
if(!fileExist(net.c_str()))
FatalError("The given network does not exist. Create the rt first.");
#ifdef __linux__
std::string input = YAMLgetConf<std::string>(conf, "input", "../demo/yolo_test.mp4");
#elif _WIN32
std::string input = YAMLgetConf<std::string>(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4");
#endif
if(!fileExist(input.c_str()))
FatalError("The given input video does not exist.");
char ntype = YAMLgetConf<char>(conf, "ntype", 'y');
int n_classes = YAMLgetConf<int>(conf, "n_classes", 80);
int n_batch = YAMLgetConf<int>(conf, "n_batch", 1);
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
float conf_thresh = YAMLgetConf<float>(conf, "conf_thresh", 0.3);
bool show = YAMLgetConf<bool>(conf, "show", true);
bool save = YAMLgetConf<bool>(conf, "save", false);
std::cout <<"Net settings - net: "<< net
<<", ntype: "<< ntype
<<", n_classes: "<< n_classes
<<", n_batch: "<< n_batch
<<", conf_thresh: "<< conf_thresh<<"\n";
std::cout <<"Demo settings - input: "<< input
<<", show: "<< show
<<", save: "<< save<<"\n\n";
// create detection network
if(!show)
SAVE_RESULT = true;
tk::dnn::Yolo3Detection yolo;
tk::dnn::CenternetDetection cnet;
tk::dnn::MobilenetDetection mbnet;
tk::dnn::MobilenetDetection mbnet;
tk::dnn::DetectionNN *detNN;
@@ -86,9 +69,10 @@ int main(int argc, char *argv[]) {
FatalError("Network type not allowed (3rd parameter)\n");
}
detNN->init(net,n_classes,n_batch,conf_thresh);
detNN->init(net, n_classes, n_batch);
gRun = true;
// open video stream
cv::VideoCapture cap(input);
if(!cap.isOpened())
gRun = false;
@@ -96,21 +80,19 @@ int main(int argc, char *argv[]) {
std::cout<<"camera started\n";
cv::VideoWriter resultVideo;
if(save) {
if(SAVE_RESULT) {
int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
cv::Mat frame;
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
cv::Mat frame;
std::vector<cv::Mat> batch_frame;
std::vector<cv::Mat> batch_dnn_input;
// start detection loop
gRun = true;
while(gRun) {
batch_dnn_input.clear();
batch_frame.clear();
@@ -138,18 +120,19 @@ int main(int argc, char *argv[]) {
cv::waitKey(1);
}
}
if(n_batch == 1 && save)
if(n_batch == 1 && SAVE_RESULT)
resultVideo << frame;
}
std::cout<<"detection end\n";
double mean = 0;
std::cout<<COL_GREENB<<"\n\nTime stats:\n";
std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";
for(int i=0; i<detNN->stats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size();
std::cout<<"Avg: "<<mean<<" ms\t"<<1000/(mean)<<" FPS\n"<<COL_END;
std::cout<<"Avg: "<<mean/n_batch<<" ms\t"<<1000/(mean/n_batch)<<" FPS\n"<<COL_END;
return 0;
}
demo/demo/demo3D.cpp
-159
View File
@@ -1,159 +0,0 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
//#include <unistd.h>
#include <mutex>
#include "demo_utils.h"
#include "CenternetDetection3D.h"
bool gRun;
bool SAVE_RESULT = false;
void sig_handler(int signo) {
std::cout<<"request gateway stop\n";
gRun = false;
}
int main(int argc, char *argv[]) {
std::cout<<"detection\n";
signal(SIGINT, sig_handler);
std::string net = "dla34_cnet3d_fp32.rt";
if(argc > 1)
net = argv[1];
#ifdef __linux__
std::string input = "../demo/yolo_test.mp4";
#elif _WIN32
std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
#endif
if(argc > 2)
input = argv[2];
std::string calib_params = "";
if(argc > 3)
calib_params = argv[3];
char ntype = 'c';
if(argc > 4)
ntype = argv[4][0];
int n_classes = 3;
if(argc > 5)
n_classes = atoi(argv[5]);
int n_batch = 1;
if(argc > 6)
n_batch = atoi(argv[6]);
bool show = true;
if(argc > 7)
show = atoi(argv[7]);
float conf_thresh=0.3;
if(argc > 8)
conf_thresh = atof(argv[8]);
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
if(!show)
SAVE_RESULT = true;
tk::dnn::CenternetDetection3D cnet;
tk::dnn::DetectionNN3D *detNN;
switch(ntype)
{
case 'c':
detNN = &cnet;
break;
default:
FatalError("Network type not allowed (3rd parameter)\n");
}
std::vector<cv::Mat> calibs;
if(!calib_params.empty() && calib_params!="NULL") {
std::cout<<"calib_params: "<<calib_params<<std::endl;
cv::Mat calib;
// the calibration matrix must be a 3x3 matrix
readCalibrationMatrix(calib_params, calib);
for(int bi=0; bi< n_batch; ++bi)
calibs.push_back(calib);
}
detNN->init(net, n_classes, n_batch, conf_thresh, calibs);
gRun = true;
cv::VideoCapture cap(input);
if(!cap.isOpened())
gRun = false;
else
std::cout<<"camera started\n";
cv::VideoWriter resultVideo;
if(SAVE_RESULT) {
int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
cv::Mat frame;
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
std::vector<cv::Mat> batch_frame;
std::vector<cv::Mat> batch_dnn_input;
while(gRun) {
batch_dnn_input.clear();
batch_frame.clear();
for(int bi=0; bi< n_batch; ++bi){
cap >> frame;
if(!frame.data)
break;
batch_frame.push_back(frame);
// this will be resized to the net format
batch_dnn_input.push_back(frame.clone());
}
if(!frame.data)
break;
//inference
detNN->update(batch_dnn_input, n_batch, false, nullptr, false);
detNN->draw(batch_frame);
if(show){
for(int bi=0; bi< n_batch; ++bi){
cv::imshow("detection", batch_frame[bi]);
cv::waitKey(1);
}
}
if(n_batch == 1 && SAVE_RESULT)
resultVideo << frame;
}
std::cout<<"detection end\n";
double mean = 0;
std::cout<<COL_GREENB<<"\n\nTime preprocessing stats:\n";
std::cout<<"Min: "<<*std::min_element(detNN->pre_stats.begin(), detNN->pre_stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->pre_stats.begin(), detNN->pre_stats.end())<<" ms\n";
for(int i=0; i<detNN->pre_stats.size(); i++) mean += detNN->pre_stats[i]; mean /= detNN->pre_stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
mean=0;
std::cout<<COL_GREENB<<"\n\nTime stats:\n";
std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
for(int i=0; i<detNN->stats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
mean=0;
std::cout<<COL_GREENB<<"\n\nTime postprocessing stats:\n";
std::cout<<"Min: "<<*std::min_element(detNN->post_stats.begin(), detNN->post_stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->post_stats.begin(), detNN->post_stats.end())<<" ms\n";
for(int i=0; i<detNN->post_stats.size(); i++) mean += detNN->post_stats[i]; mean /= detNN->post_stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
return 0;
}
demo/demo/demoDepth.cpp
-106
View File
@@ -1,106 +0,0 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
//#include <unistd.h>
#include <mutex>
#include "tkDNN/DepthNN.h"
bool gRun;
void sig_handler(int signo) {
std::cout<<"request gateway stop\n";
gRun = false;
}
int main(int argc, char *argv[]) {
signal(SIGINT, sig_handler);
std::string net = "monodepth2_fp32.rt";
if(argc > 1)
net = argv[1];
#ifdef __linux__
std::string input = "../demo/yolo_test.mp4";
#elif _WIN32
std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
#endif
if(argc > 2)
input = argv[2];
bool show = true;
if(argc > 3)
show = atoi(argv[3]);
bool save = true;
if(argc > 4)
save = atoi(argv[4]);
std::cout <<"Net settings - net: "<< net
<<"\n";
std::cout <<"Demo settings - input: "<< input
<<", show: "<< show
<<", save: "<< save<<"\n\n";
tk::dnn::DepthNN depthNN;
// create depth network
int n_batch = 1;
depthNN.init(net, n_batch);
// open video stream
cv::VideoCapture cap(input);
if(!cap.isOpened())
gRun = false;
else
std::cout<<"camera started\n";
cv::VideoWriter resultVideo;
if(save) {
int w = depthNN.output_w;
int h = depthNN.output_h;
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
if(show)
cv::namedWindow("depth", cv::WINDOW_NORMAL);
cv::Mat frame;
std::vector<cv::Mat> batch_frame;
std::vector<cv::Mat> batch_dnn_input;
// start detection loop
gRun = true;
while(gRun) {
batch_dnn_input.clear();
batch_frame.clear();
//read frame
cap >> frame;
if(!frame.data)
break;
batch_frame.push_back(frame);
batch_dnn_input.push_back(frame.clone());
//inference
depthNN.update(batch_dnn_input, 1);
if(show){
cv::imshow("depth", depthNN.depthMats[0]);
cv::waitKey(1);
}
if(save)
resultVideo << depthNN.depthMats[0];
}
std::cout<<"detection end\n";
double mean = 0;
std::cout<<COL_GREENB<<"\n\nTime stats depth:\n";
std::cout<<"Min: "<<*std::min_element(depthNN.stats.begin(), depthNN.stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(depthNN.stats.begin(), depthNN.stats.end())<<" ms\n";
for(int i=0; i<depthNN.stats.size(); i++) mean += depthNN.stats[i]; mean /= depthNN.stats.size();
std::cout<<"Avg: "<<mean<<" ms\t"<<1000/(mean)<<" FPS\n";
return 0;
}
demo/demo/demoTracker.cpp
-160
View File
@@ -1,160 +0,0 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
//#include <unistd.h>
#include <mutex>
#include "demo_utils.h"
#include "CenterTrack.h"
bool gRun;
bool SAVE_RESULT = false;
void sig_handler(int signo) {
std::cout<<"request gateway stop\n";
gRun = false;
}
int main(int argc, char *argv[]) {
std::cout<<"detection\n";
signal(SIGINT, sig_handler);
std::string net = "dla34_cnet3d_track_fp32.rt";
if(argc > 1)
net = argv[1];
#ifdef __linux__
std::string input = "../demo/yolo_test.mp4";
#elif _WIN32
std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
#endif
if(argc > 2)
input = argv[2];
std::string calib_params = "";
if(argc > 3)
calib_params = argv[3];
char ntype = 'c';
if(argc > 4)
ntype = argv[4][0];
int n_classes = 3;
if(argc > 5)
n_classes = atoi(argv[5]);
int n_batch = 1;
if(argc > 6)
n_batch = atoi(argv[6]);
bool show = true;
if(argc > 7)
show = atoi(argv[7]);
float conf_thresh=0.3;
if(argc > 8)
conf_thresh = atof(argv[8]);
bool t3d = true;
if(argc > 9)
t3d = atoi(argv[9]);
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
if(!show)
SAVE_RESULT = true;
tk::dnn::CenterTrack ctrack;
tk::dnn::TrackingNN *trackNN;
switch(ntype)
{
case 'c':
trackNN = &ctrack;
break;
default:
FatalError("Network type not allowed (3rd parameter)\n");
}
std::vector<cv::Mat> calibs;
if(!calib_params.empty() && calib_params!="NULL") {
std::cout<<"calib_params: "<<calib_params<<std::endl;
cv::Mat calib;
// the calibration matrix must be a 3x3 matrix
readCalibrationMatrix(calib_params, calib);
for(int bi=0; bi< n_batch; ++bi)
calibs.push_back(calib);
}
trackNN->init(net, n_classes, n_batch, conf_thresh, t3d, calibs);
gRun = true;
cv::VideoCapture cap(input);
if(!cap.isOpened())
gRun = false;
else
std::cout<<"camera started\n";
cv::VideoWriter resultVideo;
if(SAVE_RESULT) {
int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
cv::Mat frame;
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
std::vector<cv::Mat> batch_frame;
std::vector<cv::Mat> batch_dnn_input;
while(gRun) {
batch_dnn_input.clear();
batch_frame.clear();
for(int bi=0; bi< n_batch; ++bi){
cap >> frame;
if(!frame.data)
break;
batch_frame.push_back(frame);
// this will be resized to the net format
batch_dnn_input.push_back(frame.clone());
}
if(!frame.data)
break;
//inference
trackNN->update(batch_dnn_input, n_batch, false, nullptr, false);
trackNN->draw(batch_frame);
if(show){
for(int bi=0; bi< n_batch; ++bi){
cv::imshow("detection", batch_frame[bi]);
cv::waitKey(1);
}
}
if(n_batch == 1 && SAVE_RESULT)
resultVideo << frame;
}
std::cout<<"detection end\n";
double mean = 0;
std::cout<<COL_GREENB<<"\n\nTime preprocessing stats:\n";
std::cout<<"Min: "<<*std::min_element(trackNN->pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(trackNN->pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n";
for(int i=0; i<trackNN->pre_stats.size(); i++) mean += trackNN->pre_stats[i]; mean /= trackNN->pre_stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
mean=0;
std::cout<<COL_GREENB<<"\n\nTime stats:\n";
std::cout<<"Min: "<<*std::min_element(trackNN->stats.begin(), trackNN->stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(trackNN->stats.begin(), trackNN->stats.end())<<" ms\n";
for(int i=0; i<trackNN->stats.size(); i++) mean += trackNN->stats[i]; mean /= trackNN->stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
mean=0;
std::cout<<COL_GREENB<<"\n\nTime postprocessing stats:\n";
std::cout<<"Min: "<<*std::min_element(trackNN->post_stats.begin(), trackNN->post_stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(trackNN->post_stats.begin(), trackNN->post_stats.end())<<" ms\n";
for(int i=0; i<trackNN->post_stats.size(); i++) mean += trackNN->post_stats[i]; mean /= trackNN->post_stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
return 0;
}
demo/demo/map.cpp
+78 -107
View File
@@ -2,10 +2,7 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
@@ -32,19 +29,18 @@ int main(int argc, char *argv[])
{
char ntype = 'y';
const char *config_filename = "../demo/config.yaml";
const char * net = "yolo4tiny_fp32.rt";
const char * net = "yolo3.rt";
const char * labels_path = "../demo/COCO_val2017/all_labels.txt";
int n_batches = 1;
float confidence_thresh = 0.3;
bool show = false;
bool write_dets = false;
bool write_res_on_file = true;
bool write_coco_json = false;
bool write_coco_json = true;
int n_images = 5000;
bool verbose;
int classes, map_points, map_levels;
float map_step, IoU_thresh, conf_thresh;
double vm_total = 0, rss_total = 0;
double vm, rss;
@@ -52,17 +48,11 @@ int main(int argc, char *argv[])
if(argc > 1)
net = argv[1];
if(argc > 2)
ntype = argv[2][0];
ntype = argv[2][0];
if(argc > 3)
labels_path = argv[3];
if(argc > 4)
config_filename = argv[4];
if(argc > 5)
n_batches = atoi(argv[5]);
if(argc > 6)
confidence_thresh = atof(argv[6]);
std::cout<<"conf t: "<<confidence_thresh<<std::endl;
//check if files needed exist
if(!fileExist(config_filename))
@@ -90,9 +80,9 @@ int main(int argc, char *argv[])
}
if(write_res_on_file){
times.open("times_"+net_name+"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh)+".csv");
times.open("times_"+net_name+".csv");
memory.open("memory.csv", std::ios_base::app);
memory<<net_name+"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh)<<";";
memory<<net<<";";
}
// instantiate detector
@@ -115,7 +105,7 @@ int main(int argc, char *argv[])
default:
FatalError("Network type not allowed (3rd parameter)\n");
}
detNN->init(net,n_classes, 1, conf_thresh);
detNN->init(net, n_classes);
//read images
std::ifstream all_labels(labels_path);
@@ -128,109 +118,90 @@ int main(int argc, char *argv[])
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
bool file_ok = false;
int images_done;
for (images_done=0 ; images_done < n_images ;) {
for (images_done=0 ; std::getline(all_labels, l_filename) && images_done < n_images ; ++images_done) {
std::cout <<COL_ORANGEB<< "Images done:\t" << images_done<< "\n"<<COL_END;
int cur_batches = 0;
tk::dnn::Frame f;
f.lFilename = l_filename;
f.iFilename = l_filename;
convertFilename(f.iFilename, "labels", "images", ".txt", ".jpg");
// read frame
if(!fileExist(f.iFilename.c_str()))
FatalError("Wrong image file path.");
cv::Mat frame = cv::imread(f.iFilename.c_str(), cv::IMREAD_COLOR);
std::vector<cv::Mat> batch_frames;
std::vector<cv::Mat> batch_dnn_input;
std::vector<tk::dnn::Frame> cur_frames;
for(;cur_batches<n_batches && images_done < n_images;cur_batches++, ++images_done){
batch_frames.push_back(frame);
int height = frame.rows;
int width = frame.cols;
std::getline(all_labels, l_filename);
file_ok = all_labels ? true : false ;
if (!file_ok)
break;
tk::dnn::Frame f;
f.lFilename = l_filename;
f.iFilename = l_filename;
convertFilename(f.iFilename, "labels", "images", ".txt", ".jpg");
// read frame
if(!fileExist(f.iFilename.c_str()))
FatalError("Wrong image file path.");
cv::Mat frame = cv::imread(f.iFilename.c_str(), cv::IMREAD_COLOR);
batch_frames.push_back(frame);
f.height = frame.rows;
f.width = frame.cols;
if(!frame.data)
break;
batch_dnn_input.push_back(frame.clone());
// read and save groundtruth labels
if(fileExist(f.lFilename.c_str()))
{
std::ifstream labels(f.lFilename);
for(std::string line; std::getline(labels, line); ){
std::istringstream in(line);
tk::dnn::BoundingBox b;
in >> b.cl >> b.x >> b.y >> b.w >> b.h;
b.prob = 1;
b.truthFlag = 1;
f.gt.push_back(b);
if(show)// draw rectangle for groundtruth
cv::rectangle(batch_frames[cur_batches], cv::Point((b.x-b.w/2)*f.width, (b.y-b.h/2)*f.height), cv::Point((b.x+b.w/2)*f.width,(b.y+b.h/2)*f.height), cv::Scalar(0, 255, 0), 2);
}
}
cur_frames.push_back(f);
}
if (!file_ok)
if(!frame.data)
break;
std::vector<cv::Mat> batch_dnn_input;
batch_dnn_input.push_back(frame.clone());
//inference
detNN->update(batch_dnn_input,cur_batches,write_res_on_file, &times, write_coco_json);
detected_bbox.clear();
detNN->update(batch_dnn_input,1,write_res_on_file, &times, write_coco_json);
detNN->draw(batch_frames);
detected_bbox = detNN->detected;
for(int j=0;j<cur_frames.size(); ++j){
if(write_coco_json)
printJsonCOCOFormat(&coco_json, cur_frames[j].iFilename.c_str(), detNN->batchDetected[j], classes, cur_frames[j].width, cur_frames[j].height);
if(write_coco_json)
printJsonCOCOFormat(&coco_json, f.iFilename.c_str(), detected_bbox, classes, width, height);
std::ofstream myfile;
if(write_dets)
myfile.open ("det/"+cur_frames[j].lFilename.substr(cur_frames[j].lFilename.find("labels/") + 7));
std::ofstream myfile;
if(write_dets)
myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("labels/") + 7));
// save detections labels
for(auto d:detNN->batchDetected[j]){
//convert detected bb in the same format as label
//<x_center>/<image_width> <y_center>/<image_width> <width>/<image_width> <height>/<image_width>
tk::dnn::BoundingBox b;
b.x = (d.x + d.w/2) / cur_frames[j].width;
b.y = (d.y + d.h/2) / cur_frames[j].height;
b.w = d.w / cur_frames[j].width;
b.h = d.h / cur_frames[j].height;
b.prob = d.prob;
b.cl = d.cl;
cur_frames[j].det.push_back(b);
if(write_dets)
myfile << d.cl << " "<< d.prob << " "<< b.x << " "<< b.y << " "<< b.w << " "<< b.h <<"\n";
if(show)// draw rectangle for detection
cv::rectangle(batch_frames[j], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2);
}
// save detections labels
for(auto d:detected_bbox){
//convert detected bb in the same format as label
//<x_center>/<image_width> <y_center>/<image_width> <width>/<image_width> <height>/<image_width>
tk::dnn::BoundingBox b;
b.x = (d.x + d.w/2) / width;
b.y = (d.y + d.h/2) / height;
b.w = d.w / width;
b.h = d.h / height;
b.prob = d.prob;
b.cl = d.cl;
f.det.push_back(b);
if(write_dets)
myfile.close();
images.push_back(cur_frames[j]);
if(show){
cv::imshow("detection", batch_frames[j]);
cv::waitKey(0);
}
myfile << d.cl << " "<< d.prob << " "<< b.x << " "<< b.y << " "<< b.w << " "<< b.h <<"\n";
if(show)// draw rectangle for detection
cv::rectangle(batch_frames[0], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2);
}
std::cout <<COL_ORANGEB<< "Images done:\t" << images_done<< "\tcur batch:\t"<<cur_batches<< "\n"<<COL_END;
if(write_dets)
myfile.close();
// read and save groundtruth labels
if(fileExist(f.lFilename.c_str()))
{
std::ifstream labels(l_filename);
for(std::string line; std::getline(labels, line); ){
std::istringstream in(line);
tk::dnn::BoundingBox b;
in >> b.cl >> b.x >> b.y >> b.w >> b.h;
b.prob = 1;
b.truthFlag = 1;
f.gt.push_back(b);
if(show)// draw rectangle for groundtruth
cv::rectangle(batch_frames[0], cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2);
}
}
images.push_back(f);
if(show){
cv::imshow("detection", batch_frames[0]);
cv::waitKey(0);
}
getMemUsage(vm, rss);
vm_total += vm;
rss_total += rss;
@@ -247,11 +218,11 @@ int main(int argc, char *argv[])
std::cout << "Avg VM[MB]: " << vm_total/images_done/1024.0 << ";Avg RSS[MB]: " << rss_total/images_done/1024.0 << std::endl;
//compute mAP
double AP = tk::dnn::computeMapNIoULevels(images,classes,IoU_thresh,confidence_thresh, map_points, map_step, map_levels, verbose, write_res_on_file, net_name+"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh));
double AP = tk::dnn::computeMapNIoULevels(images,classes,IoU_thresh,conf_thresh, map_points, map_step, map_levels, verbose, write_res_on_file, net_name);
std::cout<<"mAP "<<IoU_thresh<<":"<<IoU_thresh+map_step*(map_levels-1)<<" = "<<AP<<std::endl;
//compute average precision, recall and f1score
tk::dnn::computeTPFPFN(images,classes,IoU_thresh,confidence_thresh, verbose, write_res_on_file, net_name +"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh));
tk::dnn::computeTPFPFN(images,classes,IoU_thresh,conf_thresh, verbose, write_res_on_file, net_name);
if(write_res_on_file){
memory<<vm_total/images_done/1024.0<<";"<<rss_total/images_done/1024.0<<"\n";
demo/demo/seg_demo.cpp
-150
View File
@@ -1,150 +0,0 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "SegmentationNN.h"
bool gRun;
bool SAVE_RESULT = true;
void sig_handler(int signo) {
std::cout<<"request gateway stop\n";
gRun = false;
}
void writePred(const std::string& images_names, const std::string& gt_folder, const std::string& out_folder, tk::dnn::SegmentationNN& segNN, int& width, int& height, bool show=false){
std::ifstream all_gt(images_names);
std::string filename;
cv::Mat frame;
for (; std::getline(all_gt, filename); ) {
std::cout<<filename<<std::endl;
frame = cv::imread(gt_folder + filename);
height = frame.rows;
width = frame.cols;
segNN.updateOriginal(frame, false);
if(show)
segNN.draw();
cv::imwrite(out_folder + filename, segNN.segmented[0]);
}
}
int main(int argc, char *argv[]) {
std::cout<<"detection\n";
signal(SIGINT, sig_handler);
std::string net = "shelfnet_fp32.rt";
if(argc > 1)
net = argv[1];
std::string input = "../demo/yolo_test.mp4";
if(argc > 2)
input = argv[2];
int n_batch = 1;
if(argc > 3)
n_batch = atoi(argv[3]);
int n_classes = 19;
if(argc > 4)
n_classes = atoi(argv[4]);
bool resize = false;
if(argc > 5)
resize = atoi(argv[5]);
int baseline_resize = 1024;
if(argc > 6)
baseline_resize = atoi(argv[6]);
bool show = true;
if(argc > 7)
show = atoi(argv[7]);
bool write_pred = false;
if(argc > 8)
write_pred = atoi(argv[8]);
if(resize && (baseline_resize < 0 || baseline_resize > 5000))
FatalError("Problem with baseline resize")
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
//net initialization
tk::dnn::SegmentationNN segNN;
segNN.init(net, n_classes, n_batch);
int height = 0, width = 0;
int basewidth=baseline_resize, hsize;
if(write_pred){
std::string gt_folder = "../demo/CityScapes_val/images/";
std::string images_names = "../demo/CityScapes_val/all_images.txt";
std::string out_folder = "seg/";
writePred(images_names, gt_folder, out_folder, segNN, width, height, show);
}
else{
if(!show)
SAVE_RESULT = true;
gRun = true;
cv::VideoCapture cap(input);
if(!cap.isOpened())
gRun = false;
else
std::cout<<"camera started\n";
cv::VideoWriter resultVideo;
if(SAVE_RESULT) {
int w,h;
if(resize){
w = basewidth;
h = int((float(cap.get(cv::CAP_PROP_FRAME_HEIGHT))*float(basewidth/float(cap.get(cv::CAP_PROP_FRAME_WIDTH)))));
}
else{
w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
}
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
cv::Mat frame;
while(gRun) {
cap >> frame;
if(!frame.data)
break;
if(resize){
hsize = int((float(frame.rows)*float(basewidth/float(frame.cols))));
cv::resize(frame, frame, cv::Size(basewidth, hsize));
}
height = frame.rows;
width = frame.cols;
//inference
segNN.updateOriginal(frame, true);
if(show)
segNN.draw();
if(SAVE_RESULT)
resultVideo << segNN.segmented[0];
}
}
std::cout<<"segmentation end\n";
double mean = 0, mean_pre = 0, mean_post = 0;
std::cout<<COL_GREENB<<"\n\nTime stats for size ["<<width<<","<<height<<"] :\n";
for(int i=0; i<segNN.stats.size(); i++) mean += segNN.stats[i]; mean /= segNN.stats.size();
for(int i=0; i<segNN.stats_pre.size(); i++) mean_pre += segNN.stats_pre[i]; mean_pre /= segNN.stats_pre.size();
for(int i=0; i<segNN.stats_post.size(); i++) mean_post += segNN.stats_post[i]; mean_post /= segNN.stats_post.size();
std::cout<<"Avg pre:\t"<<mean_pre<<" ms\t"<<1000/(mean_pre)<<" FPS\n";
std::cout<<"Avg inf:\t"<<mean<<" ms\t"<<1000/(mean)<<" FPS\n";
std::cout<<"Avg post:\t"<<mean_post<<" ms\t"<<1000/(mean_post)<<" FPS\n\n";
std::cout<<"Avg tot:\t"<<(mean_pre + mean_post + mean) <<" ms\t"<<1000/((mean_pre + mean_post + mean))<<" FPS\n"<<COL_END;
return 0;
}
demo/demoConfig.yaml
-14
View File
@@ -1,14 +0,0 @@
# video input
input : "../demo/yolo_test.mp4"
win_input : "..\\..\\..\\demo\\yolo_test.mp4"
# network config
net : "yolo4_berkeley_fp32.rt"
ntype : 'y'
n_classes : 80
n_batch : 1
conf_thresh : 0.3
# demo config
show : true
save : false
docker/Dockerfile.base
+39 -122
View File
@@ -1,140 +1,57 @@
FROM nvidia/cudagl:11.3.1-devel-ubuntu20.04
FROM nvidia/cuda:10.2-cudnn7-devel-ubuntu18.04
LABEL maintainer "Francesco Gatti"
LABEL maintainer "TKDNN AUTHORS"
LABEL Description="tkDNN+cudagl"
LABEL com.tkdnn.nvidia.version="11.3.1"
ADD nv-tensorrt-repo-ubuntu1804-cuda10.2-trt7.0.0.11-ga-20191216_1-1_amd64.deb /tmp/trt.deb
RUN apt-get update && dpkg -i /tmp/trt.deb && rm /tmp/trt.deb && apt-get update
RUN apt install -y libnvinfer7=7.0.0-1+cuda10.2 libnvinfer-dev=7.0.0-1+cuda10.2
RUN DEBIAN_FRONTEND=noninteractive apt install -y git wget libeigen3-dev libyaml-cpp-dev
RUN cd /tmp && \
wget https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh && \
chmod +x cmake-3.17.3-Linux-x86_64.sh && \
./cmake-3.17.3-Linux-x86_64.sh --prefix=/usr/local --exclude-subdir --skip-license && \
rm ./cmake-3.17.3-Linux-x86_64.sh
ENV DEBIAN_FRONTEND noninteractive
ENV CC gcc
ENV CXX g++
RUN apt-get update && apt-get install -y \
libblkid-dev && apt-get clean && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install -y \
libcudnn8-dev=8.2.1.32-1+cuda11.3 \
libcudnn8=8.2.1.32-1+cuda11.3 \
libnvinfer-dev=8.0.3-1+cuda11.3 \
libnvinfer8=8.0.3-1+cuda11.3 && apt-get clean && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install -y --no-install-recommends \
libblkid-dev \
locales \
lsb-release \
mesa-utils \
git \
nano \
terminator \
wget \
curl \
libssl-dev \
htop \
dbus-x11 \
libqt5opengl5-dev \
libgtk-3-dev \
libvtk7-dev \
libv4l-dev \
tar \
libgoogle-glog-dev \
libgflags-dev \
gfortran-9 \
libtbb-dev \
libgstreamer1.0-dev \
libgstreamer-plugins-base1.0-dev \
libdc1394-22-dev \
libavresample-dev \
libatlas-cpp-0.6-dev \
python3-dev \
gdb \
python3-pip \
unzip libtbb-dev && \
apt-get clean && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install -y --no-install-recommends \
software-properties-common && apt-get clean && rm -rf /var/lib/apt/lists/*
RUN apt-add-repository universe
RUN apt-get update && apt-get install -y python3-pip python3 openssh-server ssh pyqt5-dev sip-dev && apt-get clean && rm -rf /var/lib/apt/lists/*
RUN pip3 install --upgrade pip
RUN pip3 install --upgrade virtualenv
RUN pip3 install --upgrade paramiko
RUN pip3 install --ignore-installed --upgrade numpy protobuf
RUN cd ~ && mkdir build
RUN cd ~/build && wget https://github.com/Kitware/CMake/releases/download/v3.21.4/cmake-3.21.4.tar.gz && \
tar -xvf cmake-3.21.4.tar.gz && cd cmake-3.21.4 && ./configure --prefix=/usr/local --qt-gui --parallel=12 && \
make -j8 && make install
RUN apt-get update && apt-get install -y automake autoconf pkg-config libevent-dev libncurses5-dev bison && \
apt-get clean && rm -rf /var/lib/apt/lists/
RUN git clone https://github.com/tmux/tmux.git && \
cd tmux && git checkout tags/3.2 && ls -la && sh autogen.sh && ./configure && make -j8 && make install
RUN apt-get update && apt-get install -y zsh && apt-get clean && rm -rf /var/lib/apt/lists/*
RUN wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | zsh || true
RUN chsh -s /usr/bin/zsh root
RUN git clone https://github.com/sindresorhus/pure /root/.oh-my-zsh/custom/pure
RUN ln -s /root/.oh-my-zsh/custom/pure/pure.zsh-theme /root/.oh-my-zsh/custom/
RUN ln -s /root/.oh-my-zsh/custom/pure/async.zsh /root/.oh-my-zsh/custom/
RUN sed -i -e 's/robbyrussell/refined/g' /root/.zshrc
RUN sed -i '/plugins=(/c\plugins=(git git-flow adb pyenv tmux)' /root/.zshrc
RUN mkdir -p /root/.config/terminator/
COPY assets/terminator_config /root/.config/terminator/config
RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "/usr/local/cuda/lib64" >> /etc/ld.so.conf.d/nvidia.conf
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/lib:/usr/lib/x86_64-linux-gnu:/usr/local/lib:${LD_LIBRARY_PATH}
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility,graphics
RUN cd ~/build && wget https://github.com/opencv/opencv/archive/4.5.4.tar.gz && tar -xf 4.5.4.tar.gz && rm 4.5.4.tar.gz
RUN cd ~/build && wget https://github.com/opencv/opencv_contrib/archive/4.5.4.tar.gz && tar -xf 4.5.4.tar.gz && rm 4.5.4.tar.gz
RUN cd ~/build && \
cd opencv-4.5.4 && mkdir build && cd build && \
RUN echo "INSTALL OPENCV"
RUN apt-get install -y build-essential \
unzip \
pkg-config \
libjpeg-dev \
libpng-dev \
libtiff-dev \
libavcodec-dev \
libavformat-dev \
libswscale-dev \
libv4l-dev \
libxvidcore-dev \
libx264-dev \
libgtk-3-dev \
libatlas-base-dev \
gfortran \
libgstreamer1.0-dev \
libgstreamer-plugins-base1.0-dev \
libdc1394-22-dev \
libavresample-dev
RUN cd && wget https://github.com/opencv/opencv/archive/4.3.0.tar.gz && tar -xf 4.3.0.tar.gz && rm *.tar.gz
RUN cd && wget https://github.com/opencv/opencv_contrib/archive/4.3.0.tar.gz && tar -xf 4.3.0.tar.gz && rm *.tar.gz
RUN cd && \
cd opencv-4.3.0 && mkdir build && cd build && \
cmake -D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=/usr/local \
-D INSTALL_PYTHON_EXAMPLES=OFF \
-D INSTALL_C_EXAMPLES=OFF \
-D OPENCV_EXTRA_MODULES_PATH='~/build/opencv_contrib-4.5.4/modules' \
-D OPENCV_EXTRA_MODULES_PATH='~/opencv_contrib-4.3.0/modules' \
-D BUILD_EXAMPLES=OFF \
-D BUILD_TESTS=OFF \
-D BUILD_PERF_TESTS=OFF \
-D BUILD_DOCS=OFF \
-D WITH_CUDA=ON \
-D WITH_OPENGL=ON \
-D WITH_NVCUVID=ON \
-D CUDA_ARCH_BIN=7.2 \
-D CUDA_ARCH_PTX=7.2 \
-D CUDA_ARCH_PTX="" \
-D ENABLE_FAST_MATH=ON \
-D CUDA_FAST_MATH=ON \
-D WITH_CUBLAS=ON \
-D WITH_CUDNN=ON \
-D WITH_OPENMP=ON \
-D WITH_NONFREE=ON \
-D WITH_LIBV4L=ON \
-D WITH_GSTREAMER=ON \
-D WITH_GSTREAMER_0_10=OFF \
-D WITH_TBB=ON \
../ && make -j12 && make install && ldconfig
../ && make -j12 && make install
RUN apt clean
RUN cd ~ && rm -rf build
RUN cd ~ && mkdir Development && cd Development && \
git clone https://github.com/ceccocats/tkDNN.git && cd tkDNN && \
mkdir build && cd build && \
cmake -DCMAKE_BUILD_TYPE=Release .. && \
make -j6
RUN apt-get clean && rm -rf /var/lib/apt/lists/*
COPY assets/entrypoint_setup.sh /
ENTRYPOINT ["/entrypoint_setup.sh"]
CMD ["terminator"]
docker/README.md
+4 -1
View File
@@ -9,10 +9,13 @@ docker build -t tkdnn:build -f Dockerfile .
# make nvidia docker working
# follow this guide: https://github.com/NVIDIA/nvidia-docker
# dowload tensorrt
# from: https://developer.nvidia.com/compute/machine-learning/tensorrt/secure/7.0/7.0.0.11/local_repo/nv-tensorrt-repo-ubuntu1804-cuda10.2-trt7.0.0.11-ga-20191216_1-1_amd64.deb
# build image
docker build -t ceccocats/tkdnn:latest -f Dockerfile.base .
# run image
./docker_launch.sh
docker run -ti --gpus all --rm ceccocats/tkdnn:latest bash
```
docker/assets/entrypoint_setup.sh
-123
View File
@@ -1,123 +0,0 @@
#! /bin/bash
CMD=
# Functions
# TOOD: Check if we can use: getent passwd $USER to extract all variables
# TODO: Check for valid inputs, cause now it will go through even with bad inputs
check_envs () {
DOCKER_CUSTOM_USER_OK=true;
if [ -z ${DOCKER_USER_NAME+x} ]; then
DOCKER_CUSTOM_USER_OK=false;
return;
fi
if [ -z ${DOCKER_USER_ID+x} ]; then
DOCKER_CUSTOM_USER_OK=false;
return;
else
if ! [ -z "${DOCKER_USER_ID##[0-9]*}" ]; then
echo -e "\033[1;33mWarning: User-ID should be a number. Falling back to defaults.\033[0m"
DOCKER_CUSTOM_USER_OK=false;
return;
fi
fi
if [ -z ${DOCKER_USER_GROUP_NAME+x} ]; then
DOCKER_CUSTOM_USER_OK=false;
return;
fi
if [ -z ${DOCKER_USER_GROUP_ID+x} ]; then
DOCKER_CUSTOM_USER_OK=false;
return;
else
if ! [ -z "${DOCKER_USER_GROUP_ID##[0-9]*}" ]; then
echo -e "\033[1;33mWarning: Group-ID should be a number. Falling back to defaults.\033[0m"
DOCKER_CUSTOM_USER_OK=false;
return;
fi
fi
}
setup_env_user () {
USER=$1
USER_ID=$2
GROUP=$3
GROUP_ID=$4
## Create user
useradd -m $USER
## Copy zsh/sh configs
cp /root/.profile /home/$USER/
cp /root/.bashrc /home/$USER/
cp /root/.zshrc /home/$USER/
## Copy terminator configs
mkdir -p /home/$USER/.config/terminator
cp /root/.config/terminator/config /home/$USER/.config/terminator/config
cp /root/.config/terminator/background.png /home/$USER/.config/terminator/background.png
cp -rf /root/.oh-my-zsh /home/$USER/
cp -rf /root/tkDNN /home/$USER/
rm -rf /home/$USER/.oh-my-zsh/custom/pure.zsh-theme /home/$USER/.oh-my-zsh/custom/async.zsh
ln -s /home/$USER/.oh-my-zsh/custom/pure/pure.zsh-theme /home/$USER/.oh-my-zsh/custom/
ln -s /home/$USER/.oh-my-zsh/custom/pure/async.zsh /home/$USER/.oh-my-zsh/custom/
sed -i -e 's@ZSH=\"/root@ZSH=\"/home/$USER@g' /home/$USER/.zshrc
# Copy SSH keys & fix owner
if [ -d "/root/.ssh" ]; then
cp -rf /root/.ssh /home/$USER/
chown -R $USER:$GROUP /home/$USER/.ssh
fi
## Fix owner
chown $USER:$GROUP /home/$USER
chown -R $USER:$GROUP /home/$USER/.config
chown $USER:$GROUP /home/$USER/.profile
chown $USER:$GROUP /home/$USER/.bashrc
chown $USER:$GROUP /home/$USER/.zshrc
chown -R $USER:$GROUP /home/$USER/.oh-my-zsh
chown -R $USER:$GROUP /home/$USER/tkDNN
## This a trick to keep the evnironmental variables of root which is important!
echo "if ! [ \"$DOCKER_USER_NAME\" = \"$(id -un)\" ]; then" >> /root/.bashrc
echo " cd /home/$DOCKER_USER_NAME" >> /root/.bashrc
echo " su $DOCKER_USER_NAME" >> /root/.bashrc
echo "fi" >> /root/.bashrc
echo "if ! [ \"$DOCKER_USER_NAME\" = \"$(id -un)\" ]; then" >> /root/.zshrc
echo " cd /home/$DOCKER_USER_NAME" >> /root/.zshrc
echo " su $DOCKER_USER_NAME" >> /root/.zshrc
echo "fi" >> /root/.zshrc
## Setup Password-file
PASSWDCONTENTS=$(grep -v "^${USER}:" /etc/passwd)
GROUPCONTENTS=$(grep -v -e "^${GROUP}:" -e "^docker:" /etc/group)
(echo "${PASSWDCONTENTS}" && echo "${USER}:x:$USER_ID:$GROUP_ID::/home/$USER:/bin/bash") > /etc/passwd
(echo "${GROUPCONTENTS}" && echo "${GROUP}:x:${GROUP_ID}:") > /etc/group
(if test -f /etc/sudoers ; then echo "${USER} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers ; fi)
}
# ---Main---
# Create new user
## Check Inputs
check_envs
## Determine user & Setup Environment
if [ $DOCKER_CUSTOM_USER_OK == true ]; then
echo " -->DOCKER_USER Input is set to '$DOCKER_USER_NAME:$DOCKER_USER_ID:$DOCKER_USER_GROUP_NAME:$DOCKER_USER_GROUP_ID'";
echo -e "\033[0;32mSetting up environment for user=$DOCKER_USER_NAME\033[0m"
setup_env_user $DOCKER_USER_NAME $DOCKER_USER_ID $DOCKER_USER_GROUP_NAME $DOCKER_USER_GROUP_ID
else
echo " -->DOCKER_USER* variables not set. Using 'root'.";
echo -e "\033[0;32mSetting up environment for user=root\033[0m"
DOCKER_USER_NAME="root"
fi
# Change shell to zsh
chsh -s /usr/bin/zsh $DOCKER_USER_NAME
# Run CMD from Docker
"$@"
docker/assets/terminator_config
-18
View File
@@ -1,18 +0,0 @@
[global_config]
title_transmit_bg_color = "#2e3436"
[keybindings]
[layouts]
[[default]]
[[[child1]]]
parent = window0
type = Terminal
[[[window0]]]
parent = ""
type = Window
[plugins]
[profiles]
[[default]]
background_color = "#282828"
cursor_color = "#aaaaaa"
foreground_color = "#f3f3f3"
palette = "#000000:#aa0000:#00aa00:#c4a000:#3465a4:#75507b:#06989a:#d3d7cf:#88807c:#f15d22:#73c48f:#ffce51:#48b9c7:#ad7fa8:#34e2e2:#eeeeec"
docker/docker_launch.sh
-9
View File
@@ -1,9 +0,0 @@
xhost local:root
docker run --rm -it --runtime=nvidia --privileged --net=host --cap-add sys_ptrace -d --ipc=host \
-v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$DISPLAY \
-v $HOME/.Xauthority:/home/$(id -un)/.Xauthority -e XAUTHORITY=/home/$(id -un)/.Xauthority \
-e DOCKER_USER_NAME=$(id -un) \
-e DOCKER_USER_ID=$(id -u) \
-e DOCKER_USER_GROUP_NAME=$(id -gn) \
-e DOCKER_USER_GROUP_ID=$(id -g) \
-v $HOME/.ssh:/home/$(id -un)/.ssh ceccocats/tkdnn
docs/README_2d3dtracking.md
-92
View File
@@ -1,92 +0,0 @@
# 2D/3D Object Detection and Tracking
Currently tkDNN supports only CenterTrack as 3DOD & 2D/3D Tracker network.
## 3D Object Detection
To run the 3D object detection demo follow these steps (example with CenterNet based on DLA34):
```
rm dla34_cnet3d_fp32.rt # be sure to delete(or move) old tensorRT files
./test_dla34_cnet3d # run the yolo test (is slow)
./demo3D dla34_cnet3d_fp32.rt ../demo/yolo_test.mp4 NULL c
```
The demo3D program takes the same parameters of the demo program:
```
./demo3D <network-rt-file> <path-to-video> <calibration-file> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh>
```
where
* ```<calibration-file>``` is the camera calibration file (opencv format). It is important that the file contains entry "camera_matrix" with sub-entry "rows", "cols", "data". If you do not want to pass the calibration file, pass "NULL" instead.
![demo](https://user-images.githubusercontent.com/11939259/126784875-c4285497-d369-424f-abda-58274cd747ac.gif)
## Object Detection and Tracking
To run the 3D object detection & tracking demo follow these steps (example with CenterTrack based on DLA34):
```
rm dla34_ctrack_fp32.rt # be sure to delete(or move) old tensorRT files
./test_dla34_ctrack # run the yolo test (is slow)
./demoTracker dla34_ctrack_fp32.rt ../demo/yolo_test.mp4 NULL c
```
The demoTracker program takes the same parameters of the demo program:
```
./demoTracker <network-rt-file> <path-to-video> <calibration-file> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh> <2D/3D-flag>
```
where
* ```<calibration-file>``` is the camera calibration file (opencv format). It is important that the file contains entry "camera_matrix" with sub-entry "rows", "cols", "data". If you do not want to pass the calibration file, pass "NULL" instead.
* ```<2D/3D-flag>``` if set to 0 the demo will be in the 2D mode, while if set to 1 the demo will be in the 3D mode (Default is 1 - 3D mode).
![demo](https://user-images.githubusercontent.com/11939259/126784878-513fa9e8-864a-4c24-b4bd-199737184708.gif)
## FPS Results
Inference FPS of shelfnet with tkDNN, average of 1200 images on:
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
* Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
### 3D OD and Tracking
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | pre | 4.43883 | 225.285 | 4.42951 | 225.759 | 4.44278 | 225.084 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | inf | 9.03454 | 110.686 | 6.02013 | 166.109 | 5.31611 | 188.108 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | post | 0.96631 | 1034.87 | 0.96824 | 1032.80 | 0.95066 | 1051.90 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | tot | 14.4397 | 69.2535 | 11.4179 | 87.5818 | 10.7095 | 93.3750 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | pre | 4.60075 | 217.356 | 4.28658 | 233.286 | 4.29473 | 232.844 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | inf | 8.48365 | 117.874 | 5.25150 | 190.422 | 4.58463 | 218.120 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | post | 0.99484 | 1005.19 | 0.91776 | 1089.61 | 0.89853 | 1112.93 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | tot | 14.0792 | 71.0266 | 10.4558 | 95.6405 | 9.77788 | 102.272 |
| AGX Xavier | CenterTrack3D 512x512 (B=1) | pre | 34.9915 | 28.5784 | 33.5976 | 29.7440 | 34.4425 | 29.0339 |
| AGX Xavier | CenterTrack3D 512x512 (B=1) | inf | 76.3579 | 13.0962 | 52.4759 | 19.0564 | 51.4610 | 19.4322 |
| AGX Xavier | CenterTrack3D 512x512 (B=1) | post | 3.38576 | 295.355 | 3.26010 | 306.739 | 3.19770 | 312.725 |
| AGX Xavier | CenterTrack3D 512x512 (B=1) | tot | 114.735 | 8.71574 | 89.3336 | 11.1940 | 89.1012 | 11.2232 |
| AGX Xavier | CenterTrack3D 512x512 (B=4) | pre | 32.8933 | 30.4014 | 32.7950 | 30.4925 | 32.9603 | 30.3396 |
| AGX Xavier | CenterTrack3D 512x512 (B=4) | inf | 74.2840 | 13.4618 | 50.3858 | 19.8469 | 49.2030 | 20.3240 |
| AGX Xavier | CenterTrack3D 512x512 (B=4) | post | 3.14888 | 317.574 | 3.13615 | 318.862 | 3.02550 | 330.524 |
| AGX Xavier | CenterTrack3D 512x512 (B=4) | tot | 110.326 | 9.06404 | 86.3169 | 11.5852 | 85.1888 | 11.7386 |
### 2D OD and Tracking
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | pre | 4.44386 | 225.030 | 4.43828 | 225.313 | 4.47747 | 223.340 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | inf | 9.08365 | 110.088 | 6.04842 | 165.332 | 5.34787 | 186.990 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | post | 0.98593 | 1014.27 | 0.97745 | 1023.07 | 0.96595 | 1035.25 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | tot | 14.5134 | 68.9018 | 11.4642 | 87.2281 | 10.7913 | 92.6672 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | pre | 4.41188 | 226.661 | 4.50800 | 221.828 | 4.29238 | 232.971 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | inf | 8.29015 | 120.625 | 5.38630 | 185.656 | 4.58500 | 218.103 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | post | 0.96847 | 1032.55 | 0.97997 | 1020.44 | 0.91791 | 1089.43 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | tot | 13.6705 | 73.1502 | 10.8743 | 91.9602 | 9.79528 | 102.090 |
| AGX Xavier | CenterTrack2D 512x512 (B=1) | pre | 33.4745 | 29.8735 | 33.4847 | 29.8643 | 33.5022 | 29.8488 |
| AGX Xavier | CenterTrack2D 512x512 (B=1) | inf | 76.2077 | 13.1220 | 52.5111 | 19.0436 | 51.6057 | 19.3777 |
| AGX Xavier | CenterTrack2D 512x512 (B=1) | post | 3.26055 | 306.697 | 3.26806 | 305.992 | 3.21988 | 310.571 |
| AGX Xavier | CenterTrack2D 512x512 (B=1) | tot | 111.943 | 8.93312 | 89.2639 | 11.2027 | 88.3278 | 11.3215 |
| AGX Xavier | CenterTrack2D 512x512 (B=4) | pre | 32.8323 | 30.4579 | 32.8595 | 30.4326 | 32.8195 | 30.4697 |
| AGX Xavier | CenterTrack2D 512x512 (B=4) | inf | 74.3075 | 13.4576 | 50.3555 | 19.8588 | 49.1805 | 20.3333 |
| AGX Xavier | CenterTrack2D 512x512 (B=4) | post | 3.12360 | 320.143 | 3.13570 | 318.908 | 3.04943 | 327.931 |
| AGX Xavier | CenterTrack2D 512x512 (B=4) | tot | 110.263 | 9.06920 | 86.3507 | 11.5807 | 85.0494 | 11.7579 |
docs/README_depth.md
-54
View File
@@ -1,54 +0,0 @@
# Monocular depth estimation with tkDNN
Currently tkDNN supports only Monodepth2 as monocular depth esitmation network.
## Run the demo
To run the depth estimation demo follow these steps (example with monodepth2):
```
rm monodepth2_fp32.rt # be sure to delete(or move) old tensorRT files
./test_monodepth2 # run the yolo test (is slow)
./demoDepth monodepth2_fp32.rt ../demo/yolo_test.mp4
```
In general the demo program takes the following parameters:
```
./demoDepth <network-rt-file> <path-to-video> <show-flag> <save-flag>
```
where
* ```<network-rt-file>``` is the rt file generated by a test
* ```<<path-to-video>``` is the path to a video file or a camera input
* ```<show-flag>``` if set to 0 the demo will not show the visualization, it will otherwise (default=1)
* ```<save-flag>``` if set to 1 the demo will save the video into result.mp4, it won't otherwise (default=1)
NB) By default it is used FP32 inference
![demo](https://user-images.githubusercontent.com/11939259/160845358-0d6ab15d-c5f4-46ae-b9da-bfaf3903389d.gif "Results on yolo_test.mp4")
<!-- ## FPS Results
Inference FPS of shelfnet with tkDNN, average of 1200 images on:
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
* Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | pre | 6.11863 | 163.435 | 5.81465 | 171.979 | 5.88699 | 169.866 |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | inf | 11.5464 | 86.6074 | 7.35396 | 135.981 | 6.37623 | 156.832 |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | post | 4.09058 | 244.464 | 3.91961 | 255.128 | 4.07343 | 245.493 |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | tot | 21.7556 | 45.9652 | 17.0882 | 58.5199 | 16.3366 | 61.2121 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | pre | 25.435 | 39.3158 | 25.2953 | 39.5331 | 25.9303 | 38.565 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | inf | 36.5015 | 27.3961 | 17.0534 | 58.6395 | 15.6061 | 64.0773 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | post | 17.3917 | 57.4985 | 17.1649 | 58.2583 | 17.5539 | 56.9675 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | tot | 79.3283 | 12.6058 | 59.5136 | 16.8029 | 59.0903 | 16.9233 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | pre | 8.0174 | 124.729 | 7.5117 | 133.126 | 7.47333 | 133.809 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | inf | 72.4173 | 13.8089 | 37.505 | 26.6631 | 31.3286 | 31.9197 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | post | 8.89958 | 112.365 | 8.83576 | 113.176 | 9.42655 | 106.083 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | tot | 89.3342 | 11.1939 | 53.8525 | 18.5692 | 48.2285 | 20.7346 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | pre | 47.1454 | 21.211 | 21.6475 | 46.1947 | 21.4201 | 46.6851 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | inf | 266.537 | 3.75183 | 128.321 | 7.79293 | 107.621 | 9.29185 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | post | 44.0711 | 22.6906 | 40.1732 | 24.8922 | 39.873 | 25.0796 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | tot | 357.753 | 2.79522 | 190.142 | 5.25922 | 168.914 | 5.92016 | -->
docs/README_seg.md
-68
View File
@@ -1,68 +0,0 @@
# Semantic Segmentation with tkDNN
Currently tkDNN supports only ShelfNet as semantic segmentation network.
## Run the demo
To run the semantic segmentation demo follow these steps (example with shelfnet):
```
rm shelfnet_fp32.rt # be sure to delete(or move) old tensorRT files
export TKDNN_BATCHSIZE=4 # be sure you have batch size > than 1 if you want to run inference on images bigger than 1024
./test_shelfnet # run the yolo test (is slow)
./demo shelfnet_fp32.rt ../demo/yolo_test.mp4 1 19
```
In general the demo program takes the following parameters:
```
./seg_demo <network-rt-file> <path-to-video> <n-batches> <number-of-classes> <resize-flag> <baseline-resize> <show-flag> <write-pred>
```
where
* ```<network-rt-file>``` is the rt file generated by a test
* ```<<path-to-video>``` is the path to a video file or a camera input
* ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
* ```<number-of-classes>```is the number of classes the network is trained on
* ```<resize-flag>``` if set to 0 the demo will not resize the input frames, but use it as it is, otherwise it will resize it.
* ```<baseline-resize>``` is ```<resize-flag>``` is set to 1, then the input frames will be proportionally resized using ```<baseline-resize>``` as width baseline.
* ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
* ```<write-pred>``` if set to 0 (default) the demo will run, otherwise the evaluation of a dataset will run and the output of the segmentation will be saved. Attention: this is under development and paths are embedded, so change them in the code in advance.
NB) By default it is used FP32 inference
NB) The batching is not used to work on more streams, rather to work on more tiles of the same image. Shelfnet never resized the input image, therefore for images greater than 1024x1024 tiles of 1024x1024 are given in input to the network in batch.
![demo](https://user-images.githubusercontent.com/11939259/126784236-38d24fc3-02df-4514-81c4-497e87e40b65.gif "Results on yolo_test.mp4")
For other demo videos refer to [this playlist](https://www.youtube.com/playlist?list=PLv0nEQYDD45y5EdSiywwCGPBmJVUzIWwe).
NB) The gif and the videos are obtained with Mapillary Vistas weights, that we cannot publicly share due to its license restrictions. However, you can train Shelfnet using Mapillary and [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of the original repo.
## FPS Results
Inference FPS of shelfnet with tkDNN, average of 1200 images on:
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
* Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | pre | 6.11863 | 163.435 | 5.81465 | 171.979 | 5.88699 | 169.866 |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | inf | 11.5464 | 86.6074 | 7.35396 | 135.981 | 6.37623 | 156.832 |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | post | 4.09058 | 244.464 | 3.91961 | 255.128 | 4.07343 | 245.493 |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | tot | 21.7556 | 45.9652 | 17.0882 | 58.5199 | 16.3366 | 61.2121 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | pre | 25.435 | 39.3158 | 25.2953 | 39.5331 | 25.9303 | 38.565 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | inf | 36.5015 | 27.3961 | 17.0534 | 58.6395 | 15.6061 | 64.0773 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | post | 17.3917 | 57.4985 | 17.1649 | 58.2583 | 17.5539 | 56.9675 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | tot | 79.3283 | 12.6058 | 59.5136 | 16.8029 | 59.0903 | 16.9233 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | pre | 8.0174 | 124.729 | 7.5117 | 133.126 | 7.47333 | 133.809 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | inf | 72.4173 | 13.8089 | 37.505 | 26.6631 | 31.3286 | 31.9197 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | post | 8.89958 | 112.365 | 8.83576 | 113.176 | 9.42655 | 106.083 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | tot | 89.3342 | 11.1939 | 53.8525 | 18.5692 | 48.2285 | 20.7346 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | pre | 47.1454 | 21.211 | 21.6475 | 46.1947 | 21.4201 | 46.6851 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | inf | 266.537 | 3.75183 | 128.321 | 7.79293 | 107.621 | 9.29185 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | post | 44.0711 | 22.6906 | 40.1732 | 24.8922 | 39.873 | 25.0796 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | tot | 357.753 | 2.79522 | 190.142 | 5.25922 | 168.914 | 5.92016 |
## Known issues
When creating the rt file all the checks returns errors. It is due to a different resize function and handling of the original ShelfNet outputs.
However, the network is supposed to work.
docs/demo.md
-120
View File
@@ -1,120 +0,0 @@
# 2D Object Detection with tkDNN
## Supported Networks
* Yolo4, Yolo4-csp, Yolo4x, Yolo4_berkeley, Yolo4tiny
* Yolo3, Yolo3_berkeley, Yolo3_coco4, Yolo3_flir, Yolo3_512, Yolo3tiny, Yolo3tiny_512
* Yolo2, Yolo2_voc, Yolo2tiny
* Csresnext50-panet-spp, Csresnext50-panet-spp_berkeley
* Resnet101_cnet, Dla34_cnet
* Mobilenetv2ssd, Mobilenetv2ssd512, Bdd-mobilenetv2ssd
## Index
- [2D Object Detection](#2d-object-detection)
- [FP16 inference](#fp16-inference)
- [INT8 inference](#int8-inference)
- [Batching](#batching)
### 2D Object Detection
This is an example using yolov4.
To run the an object detection first create the .rt file by running:
```
rm yolo4_fp32.rt # be sure to delete(or move) old tensorRT files
./test_yolo4 # run the yolo test (is slow)
```
If you get problems in the creation, try to check the error activating the debug of TensorRT in this way:
```
cmake .. -DCMAKE_BUILD_TYPE=Debug -DDEBUG=True
make
```
Once you have successfully created your rt file, run the demo:
```
./ demo <path-to-config>
```
In general the demo program takes 1 parameter, the ```<path-to-config>``` that is the path to che configuration file. The parameter is optional and its default value is ```"../demo/demoConfig.yaml"```.
The config file is a yaml file with the following attributes:
* ```net``` is the rt file generated by a test
* ```input``` is the path to a video file or a camera input (on Linux)
* ```win_input``` is the path to a video file or a camera input (on Windows)
* ```ntype``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
* ```n_classes``` is the number of classes the network is trained on
* ```n_batch``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
* ```conf_thresh``` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed.
* ```show``` if set to 0 the demo will not show the visualization (if n-batches ==1)
* ```save``` if set to 1 the demo will save the video of the demo into result.mp4 (if n-batches ==1)
N.B. By default it is used FP32 inference
![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif)
### FP16 inference
To run the demo with FP16 inference follow these steps (example with yolov3):
```
export TKDNN_MODE=FP16 # set the half floating point optimization
rm yolo4_fp16.rt # be sure to delete(or move) old tensorRT files
./test_yolo4 # run the yolo test (is slow)
#set net: yolo4_fp16.rt in the config file
./demo
```
N.B. Using FP16 inference will lead to some errors in the results (first or second decimal).
### INT8 inference
To run the demo with INT8 inference three environment variables need to be set:
* ```export TKDNN_MODE=INT8```: set the 8-bit integer optimization
* ```export TKDNN_CALIB_IMG_PATH=/path/to/calibration/image_list.txt``` : image_list.txt has in each line the absolute path to a calibration image
* ```export TKDNN_CALIB_LABEL_PATH=/path/to/calibration/label_list.txt```: label_list.txt has in each line the absolute path to a calibration label
You should provide image_list.txt and label_list.txt, using training images. However, if you want to quickly test the INT8 inference you can run (from this repo root folder)
```
bash scripts/download_validation.sh COCO
```
to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD instead of COCO to download BDD validation.
Then a complete example using yolo3 and COCO dataset would be:
```
export TKDNN_MODE=INT8
export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
rm yolo4_int8.rt # be sure to delete(or move) old tensorRT files
./test_yolo4 # run the yolo test (is slow)
#set net: yolo4_int8.rt in the config file
./demo
```
N.B.
* Using INT8 inference will lead to some errors in the results.
* The test will be slower: this is due to the INT8 calibration, which may take some time to complete.
* INT8 calibration requires TensorRT version greater than or equal to 6.0
* Only 100 images are used to create the calibration table by default (set in the code).
### Batching
#### BatchSize bigger than 1
```
export TKDNN_BATCHSIZE=2
# build tensorRT files
```
This will create a TensorRT file with the desired **max** batch size.
The test will still run with a batch of 1, but the created tensorRT can manage the desired batch size.
#### Test batch Inference
This will test the network with random input and check if the output of each batch is the same.
```
./test_rtinference <network-rt-file> <number-of-batches>
# <number-of-batches> should be less or equal to the max batch size of the <network-rt-file>
# example
export TKDNN_BATCHSIZE=4 # set max batch size
rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # build RT file
./test_rtinference yolo3_fp32.rt 4 # test with a batch size of 4
```
docs/exporting_weights.md
-127
View File
@@ -1,127 +0,0 @@
# tkDNN export weights
## Index
- [How to export weights](#how-to-export-weights)
- [1)Export weights from darknet](#1export-weights-from-darknet)
- [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101)
- [3)Export weights for CenterNet](#3export-weights-for-centernet)
- [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd)
- [5)Export weights for CenterTrack](#5export-weights-for-centertrack)
- [6)Export weights for ShelfNet](#6export-weights-for-shelfnet)
- [Darknet Parser](#darknet-parser)
## How to export weights
Weights are essential for any network to run inference. For each test a folder organized as follow is needed (in the build folder):
```
test_nn
|---- layers/ (folder containing a binary file for each layer with the corresponding wieghts and bias)
|---- debug/ (folder containing a binary file for each layer with the corresponding outputs)
```
Therefore, once the weights have been exported, the folders layers and debug should be placed in the corresponding test.
### 1)Export weights from darknet
To export weights for NNs that are defined in darknet framework, use [this](https://git.hipert.unimore.it/fgatti/darknet.git) fork of darknet and follow these steps to obtain a correct debug and layers folder, ready for tkDNN.
```
git clone https://git.hipert.unimore.it/fgatti/darknet.git
cd darknet
make
mkdir layers debug
./darknet export <path-to-cfg-file> <path-to-weights> layers
```
N.B. Use compilation with CPU (leave GPU=0 in Makefile) if you also want debug.
### 2)Export weights for DLA34 and ResNet101
To get weights and outputs needed to run the tests dla34 and resnet101 use the Python script and the Anaconda environment included in the repository.
Create Anaconda environment and activate it:
```
conda env create -f file_name.yml
source activate env_name
python <script name>
```
### 3)Export weights for CenterNet
To get the weights needed to run Centernet tests use [this](https://github.com/sapienzadavide/CenterNet.git) fork of the original Centernet.
```
git clone https://github.com/sapienzadavide/CenterNet.git
```
* follow the instruction in the README.md and INSTALL.md
```
python demo.py --input_res 512 --arch resdcn_101 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_resdcn101.pth --exp_wo --exp_wo_dim 512
python demo.py --input_res 512 --arch dla_34 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_dla_2x.pth --exp_wo --exp_wo_dim 512
```
### 4)Export weights for MobileNetSSD
To get the weights needed to run Mobilenet tests use [this](https://github.com/mive93/pytorch-ssd) fork of a Pytorch implementation of SSD network.
```
git clone https://github.com/mive93/pytorch-ssd
cd pytorch-ssd
conda env create -f env_mobv2ssd.yml
python run_ssd_live_demo.py mb2-ssd-lite <pth-model-fil> <labels-file>
```
### 5)Export weights for CenterTrack
To get the weights needed to run CenterTrack tests use [this](https://github.com/sapienzadavide/CenterTrack.git) fork of the original CenterTrack.
```
git clone https://github.com/sapienzadavide/CenterTrack.git
```
* follow the instruction in the README.md and INSTALL.md
```
python demo.py tracking,ddd --load_model ../models/nuScenes_3Dtracking.pth --dataset nuscenes --pre_hm --track_thresh 0.1 --demo /path/to/image/or/folder/or/video/or/webcam --test_focal_length 633 --exp_wo --exp_wo_dim 512 --input_h 512 --input_w 512
```
### 6)Export weights for ShelfNet
To get the weights needed to run Shelfnet tests use [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of a Pytorch implementation of Shelfnet network.
```
git clone https://git.hipert.unimore.it/mverucchi/shelfnet
cd shelfnet
cd ShelfNet18_realtime
conda env create --file shelfnet_env.yml
conda activate shelfnet
mkdir layer debug
python export.py
```
### 6)Export weights for monodepth2
To get the weights needed to run Shelfnet tests use [this](https://github.com/perseusdg/monodepth2) fork of a Pytorch implementation of monodepth2 network.
```
git clone https://github.com/perseusdg/monodepth2
cd monodepth2
mkdir models # Download the official weights and put depth.pth and encorder.pth inside this new folder
conda env create --file monodepth.yaml
conda activate monodepth2
python exporter.py # you will find the weights inside the tkDNN_bin folder
```
## Darknet Parser
tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*:
```
// example of parsing yolo4
tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names");
net->print();
```
All models from darknet are now parsed directly from cfg, you still need to export the weights with the described tools in the previous section.
<details>
<summary>Supported layers</summary>
convolutional
maxpool
avgpool
shortcut
upsample
route
reorg
region
yolo
</details>
<details>
<summary>Supported activations</summary>
relu
leaky
mish
logistic
</details>
docs/mAP_demo.md
-32
View File
@@ -1,32 +0,0 @@
# Run the mAP demo
To compute mAP, precision, recall and f1score to evaluate 2D object detectors, run the map_demo.
A validation set is needed.
To download COCO_val2017 (80 classes) run (form the root folder):
```
bash scripts/download_validation.sh COCO
```
To download Berkeley_val (10 classes) run (form the root folder):
```
bash scripts/download_validation.sh BDD
```
To compute the map, the following parameters are needed:
```
./map_demo <network rt> <network type [y|c|m]> <labels file path> <config file path>
```
where
* ```<network rt>```: rt file of a chosen network on which compute the mAP.
* ```<network type [y|c|m]>```: type of network. Right now only y(yolo), c(centernet) and m(mobilenet) are allowed
* ```<labels file path>```: path to a text file containing all the paths of the ground-truth labels. It is important that all the labels of the ground-truth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the ground-truth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg.
* ```<config file path>```: path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml
Example:
```
cd build
./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml
```
This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to submit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate).
docs/windows.md
-102
View File
@@ -1,102 +0,0 @@
# tkDNN on Windows
## Index
- [Dependencies-Windows](#dependencies-windows)
- [Compiling tkDNN on Windows](#compiling-tkdnn-on-windows)
- [Run the demo on Windows](#run-the-demo-on-windows)
- [FP16 inference windows](#fp16-inference-windows)
- [INT8 inference windows](#int8-inference-windows)
- [Run tkDNN on WSL2 with cuda](#tkdnn-on-cuda-wsl)
- [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows)
### Dependencies-Windows
This branch should work on every NVIDIA GPU supported in windows with the following dependencies:
* WINDOWS 10 1803/WINDOWS 11 or HIGHER
* CUDA 11.2
* CUDNN 8.1.1
* TENSORRT 7.2.3
* OPENCV 4.2
* MSVC 16.9+
* YAML-CPP
* EIGEN3
* 7ZIP (ADD TO PATH)
* NINJA 1.10
All the above mentioned dependencies except 7ZIP can be installed using Microsoft's [VCPKG](https://github.com/microsoft/vcpkg.git) .
After bootstrapping VCPKG the dependencies can be built and installed using the following command :
```
opencv4(normal) - vcpkg.exe install opencv4[tbb,jpeg,tiff,opengl,openmp,png,ffmpeg,eigen]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build
opencv4(cuda) - vcpkg.exe install opencv4[cuda,nonfree,contrib,eigen,tbb,jpeg,tiff,opengl,openmp,png,ffmpeg]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build
```
To build opencv4 with cuda and cudnn version corresponding to your cuda version,vcpkg's cudnn portfile needs to be modified by adding ```$ENV{CUDA_PATH}``` at lines 16 and 17 in the portfile.cmake
After VCPKG finishes building and installing all the packages delete C:\temp_vcpkg_build and add C:\opt\x64-windows\bin and C:\opt\x64-windows\debug\bin to path
### Compiling tkDNN on Windows
tkDNN is built with cmake(3.15+) on windows along with ninja.Msbuild and NMake Makefiles are drastically slower when compiling the library compared to windows
```
git clone https://github.com/ceccocats/tkDNN.git
cd tkdnn-windows
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release -G"Ninja" ..
ninja -j4
```
### Run the demo on Windows
This example uses yolo4_tiny.\
To run the object detection file create .rt file bu running:
```
.\test_yolo4tiny.exe
```
Once the rt file has been successfully create,run the demo using the following command:
```
.\demo.exe yolo4_fp32.rt ..\demo\yolo_test.mp4 y 80 ..\tests\darknet\cfg\yolo4.cfg ..\tests\darknet\names\cococ.names
```
For general info on more demo paramters,check Run the demo section on top
To run the test_all_tests.sh on windows,use git bash or msys2
### FP16 inference windows
This is an untested feature on windows.To run the object detection demo with FP16 interference follow the below steps(example with yolo4tiny):
```
set TKDNN_MODE=FP16
del /f yolo4tiny_fp16.rt
.\test_yolo4tiny.exe
.\demo.exe yolo4tiny_fp16.rt ..\demo\yolo_test.mp4
```
### INT8 inference windows
To run object detection demo with INT8 (example with yolo4tiny):
```
set TKDNN_MODE=INT8
set TKDNN_CALIB_LABEL_PATH=..\demo\COCO_val2017\all_labels.txt
set TKDNN_CALIB_IMG_PATH=..\demo\COCO_val2017\all_images.txt
del /f yolo4tiny_int8.rt # be sure to delete(or move) old tensorRT files
.\test_yolo4tiny.exe # run the yolo test (is slow)
.\demo.exe yolo4tiny_int8.rt ..\demo\yolo_test.mp4 y
```
### Run tkDNN on WSL2 with cuda
tkDNN works on wsl2 with cuda,although not all networks (centernet,mobilenet) work properly.
If you encounter issues with running the network as a result of driver not found or cuda launch error,running the following command should solve the issue
```cp /usr/lib/wsl/lib/lib* /usr/lib/x86_64-linux-gnu/ ```
### Known issues with tkDNN on Windows
In theory all models (centernet,mobilenet,darknet,centertrack,cnet3d and shelfnet) should work on Windows.
On pascal cards(sm 6x) ,nvidia cuda wsl driver 510.06 don't work well with tkDNN both on windows and cuda wsl , Nvidia drivers >465+ and < 500 are completely supported .
include/tkDNN/CenterTrack.h
-186
View File
@@ -1,186 +0,0 @@
#ifndef CENTERTRACK_H
#define CENTERTRACK_H
#include <opencv2/videoio.hpp>
#include "opencv2/opencv.hpp"
#include "kernels.h"
#include "utils.h"
#include "tkdnn.h"
#include <time.h>
#include <vector>
#include <numeric> // std::iota
#include <algorithm> // std::sort
#include "TrackingNN.h"
#ifdef _WIN32
#define _USE_MATH_DEFINES
#include <math.h>
#endif
#include "kernelsThrust.h"
namespace tk { namespace dnn {
struct detectionRes
{
float score;
int cl;
cv::Mat ct, tr, bb0, bb1;
float dep;
float dim[3];
float alpha;
float x,y,z;
float rot_y;
detectionRes() : ct(cv::Mat(cv::Size(1,2), CV_32F)),
tr(cv::Mat(cv::Size(1,2), CV_32F)),
bb0(cv::Mat(cv::Size(1,2), CV_32F)),
bb1(cv::Mat(cv::Size(1,2), CV_32F)) { }
~detectionRes() {
ct.release();
tr.release();
bb0.release();
bb1.release();
}
};
struct trackingRes
{
struct detectionRes det_res;
int tracking_id;
int age;
int active;
int color;
};
class CenterTrack : public TrackingNN
{
public:
tk::dnn::dataDim_t dim;
tk::dnn::dataDim_t dim2;
tk::dnn::dataDim_t dim_hm;
tk::dnn::dataDim_t dim_wh;
tk::dnn::dataDim_t dim_reg;
tk::dnn::dataDim_t dim_track;
tk::dnn::dataDim_t dim_dep;
tk::dnn::dataDim_t dim_rot;
tk::dnn::dataDim_t dim_dim;
tk::dnn::dataDim_t dim_amodel_offset;
/* preprocessing */
#ifdef OPENCV_CUDACONTRIB
float *mean_d;
float *stddev_d;
#else
cv::Vec<float, 3> mean;
cv::Vec<float, 3> stddev;
dnnType *input;
#endif
float *d_ptrs;
std::vector<cv::Mat> inputCalibs;
std::vector<cv::Size> szOld;
cv::Mat src;
cv::Mat dst;
cv::Mat dst2;
cv::Mat trans, trans2, transOut;
/* pre inf */
bool iter0;
dnnType *input_pre_inf_d;
bool test_pre_inf = true;
dnnType *img_d, *hm_d;
tk::dnn::dataDim_t dim_in0;
tk::dnn::dataDim_t dim_in1;
dnnType *out_d;
/* postprocessing */
int K = 100;
int width = 128;//56; // TODO
// pointer used in the kernels
float *src_out;
int *ids_out;
float *topk_scores;
int *topk_inds_;
float *topk_ys_;
float *topk_xs_;
int *ids_d, *ids_;
float *ones;
float *scores, *scores_d;
int *clses, *clses_d;
int *topk_inds_d;
float *topk_ys_d;
float *topk_xs_d;
int *inttopk_xs_d, *inttopk_ys_d;
float *bbx0, *bby0, *bbx1, *bby1;
float *bbx0_d, *bby0_d, *bbx1_d, *bby1_d;
int *intxs, *intys;
float *track, *dep, *rot, *dim_, *wh, *amodel_offset;
float *track_d, *dep_d, *rot_d, *dim_d, *wh_d, *amodel_offset_d;
float *target_coords;
/* visualization */
cv::Mat r;
std::vector<cv::Mat> calibs;
cv::Mat corners, pts3DHomo;
std::vector<std::vector<int>> faceId;
cv::Scalar trColors[256];
bool mode3D;
//processing
struct threshold op;
float outThresh = 0.1;
float newThresh = 0.3;
// float peakThreshold = 0.2;
// float centerThreshold = 0.3; //default 0.5
//detections
std::vector<struct detectionRes> detRes;
int countDet;
//tracks
std::vector<std::vector<struct trackingRes>> trRes;
std::vector<int> countTr;
std::vector<int> trackId;
bool init_preprocessing();
bool init_pre_inf();
bool init_postprocessing();
bool init_visualization(const int n_classes);
void pre_inf(const int bi);
void _get_additional_inputs();
cv::Mat transform_preds_with_trans(float x1, float x2);
void tracking(const int bi);
public:
tk::dnn::Network *pre_phase_net = nullptr;
CenterTrack() {};
~CenterTrack() {};
bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1,
const float conf_thresh=0.3, const bool mode_3d=true,
const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>());
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
void draw(std::vector<cv::Mat>& frames);
};
} // namespace dnn
} // namespace tk
#endif /*CENTERTRACK_H*/
include/tkDNN/CenternetDetection.h
+1 -1
View File
@@ -73,7 +73,7 @@ public:
CenternetDetection() {};
~CenternetDetection() {};
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3);
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
};
include/tkDNN/CenternetDetection3D.h
-106
View File
@@ -1,106 +0,0 @@
#ifndef CENTERNETDETECTION3D_H
#define CENTERNETDETECTION3D_H
#include "kernels.h"
#include <opencv2/videoio.hpp>
#include "opencv2/opencv.hpp"
#include <time.h>
#include <vector>
#include <numeric> // std::iota
#include <algorithm> // std::sort
#ifdef _WIN32
#define _USE_MATH_DEFINES
#include <math.h>
#endif
#include "DetectionNN3D.h"
#include "kernelsThrust.h"
namespace tk { namespace dnn {
class CenternetDetection3D : public DetectionNN3D
{
private:
tk::dnn::dataDim_t dim;
tk::dnn::dataDim_t dim2;
tk::dnn::dataDim_t dim_hm;
tk::dnn::dataDim_t dim_wh;
tk::dnn::dataDim_t dim_reg;
tk::dnn::dataDim_t dim_dep;
tk::dnn::dataDim_t dim_rot;
tk::dnn::dataDim_t dim_dim;
std::vector<cv::Mat> inputCalibs;
float *topk_scores;
int *topk_inds_;
float *topk_ys_;
float *topk_xs_;
int *ids_d, *ids_;
float *ones;
float *scores, *scores_d;
int *clses, *clses_d;
int *topk_inds_d;
float *topk_ys_d;
float *topk_xs_d;
int *inttopk_xs_d, *inttopk_ys_d;
float *xs, *ys;
float *dep, *rot, *dim_, *wh;
float *dep_d, *rot_d, *dim_d, *wh_d;
float *target_coords;
#ifdef OPENCV_CUDACONTRIB
float *mean_d;
float *stddev_d;
#else
cv::Vec<float, 3> mean;
cv::Vec<float, 3> stddev;
dnnType *input;
#endif
cv::Mat r;
float *d_ptrs;
cv::Size sz_old;
cv::Mat src;
cv::Mat dst;
cv::Mat dst2;
cv::Mat trans, trans2;
std::vector<cv::Mat> calibs;
//processing
int K = 100;
int width = 128;//56; // TODO
// pointer used in the kernels
float *srcOut;
int *idsOut;
struct threshold op;
cv::Mat corners, pts3DHomo;
std::vector<std::vector<int>> faceId;
public:
CenternetDetection3D() {};
~CenternetDetection3D() {};
bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>());
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
void draw(std::vector<cv::Mat>& frames);
};
} // namespace dnn
} // namespace tk
#endif /*CENTERNETDETECTION_H*/
include/tkDNN/DarknetParser.h
-6
View File
@@ -24,10 +24,7 @@ namespace tk { namespace dnn {
int num = 1;
int pad = 0;
int coords = 4;
int nms_kind = 0;
int new_coords= 0;
float scale_xy = 1;
float nms_thresh = 0.45;
std::vector<int> layers;
std::string activation = "linear";
@@ -47,8 +44,5 @@ namespace tk { namespace dnn {
std::vector<tk::dnn::Layer*> &netLayers, const std::vector<std::string>& names);
std::vector<std::string> darknetReadNames(const std::string& names_file);
tk::dnn::Network* darknetParser(const std::string& cfg_file, const std::string& wgs_path, const std::string& names_file);
void loadYoloInfo(const std::string &cfg_file,int lineNo,std::vector<float> &mask,std::vector<float> &anchors,int &num,int &classes,float &nms_thresh,int &nms_kind,int &coords);
void loadYoloInitInfo(int &channels,int &width,int &height,const std::string &cfg_file);
std::vector<int> noYolosLine(const std::string &cfg_file);
}}
include/tkDNN/DepthNN.h
-180
View File
@@ -1,180 +0,0 @@
#ifndef DEPTHNN_H
#define DEPTHNN_H
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "tkDNN/utils.h"
#include "tkDNN/tkdnn.h"
#include "NetworkViz.h"
namespace tk { namespace dnn {
class DepthNN {
public:
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_h;
dnnType *input_d;
float* depth_h;
int output_w;
int output_h;
int nBatches = 1;
cv::Mat bgr[3];
cv::Mat imagePreproc;
std::vector<double> stats; /*keeps track of inference times (ms)*/
std::vector<std::vector<float>> depths;
std::vector<cv::Mat> depthMats;
DepthNN() {};
~DepthNN(){};
/**
* Method used to initialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file of the NN.
* @param n_batches maximum number of batches to use in inference
* @return true if everything is correct, false otherwise.
*/
void init(const std::string& tensor_path, const int n_batches=1){
//create net
std::cout<<(tensor_path).c_str()<<"\n";
nBatches = n_batches;
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str());
//allocate memory for NN input
checkCuda(cudaMallocHost(&input_h, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
//allocate memory for NN output
depthMats.resize(nBatches);
depths.resize(nBatches);
for(int i=0; i< depths.size();++i)
depths[i].resize(netRT->buffersDIM[1].tot());
depth_h = (float *)malloc(netRT->buffersDIM[1].tot() * sizeof(float));
output_h = netRT->buffersDIM[1].h;
output_w = netRT->buffersDIM[1].w;
}
/**
* This method preprocess the image, before feeding it to the NN.
*
* @param frame original frame to adapt for inference.
* @param bi batch index
*/
void preprocess(cv::Mat &frame, const int bi=0) {
//resize image, remove mean, divide by std
cv::Mat frame_nomean;
resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
frame.convertTo(frame_nomean, CV_32FC3);
frame_nomean.convertTo(imagePreproc, CV_32FC3, 1 / 255.0, 0);
//copy image into tensor and copy it into GPU
cv::split(imagePreproc, bgr);
for (int i = 0; i < netRT->input_dim.c; i++){
int idx = i * imagePreproc.rows * imagePreproc.cols;
int ch = netRT->input_dim.c-1 -i;
memcpy((void *)&input_h[idx + netRT->input_dim.tot()*bi], (void *)bgr[ch].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input_h + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
}
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param bi batch index
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
void postprocess(const int bi=0) {
dnnType *rt_out[1];
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
checkCuda(cudaMemcpy(depth_h, rt_out[0], netRT->buffersDIM[1].tot()* sizeof(float), cudaMemcpyDeviceToHost));
memcpy(&depths[bi][0], &depth_h[0], netRT->buffersDIM[1].tot()* sizeof(float));
// cv::Mat d(netRT->buffersDIM[1].h, netRT->buffersDIM[1].w, CV_8UC1, depth_h);
// depthMats[bi] = d.clone();
cv::Mat depth_mat = vizData2Mat(rt_out[0], netRT->buffersDIM[1], netRT->buffersDIM[1].h, netRT->buffersDIM[1].w);
// cv::Mat depth_mat = vizData2Mat((dnnType *)netRT->buffersRT[0], netRT->buffersDIM[0], netRT->buffersDIM[0].h, netRT->buffersDIM[0].w);
depthMats[bi] = depth_mat.clone();
}
/**
* This method performs the inference of the NN.
*
* @param frames frames to build the embedding from.
* @param cur_batches number of batches to use in inference
*/
void update(std::vector<cv::Mat>& frames, const int cur_batches=1){
if(cur_batches > nBatches)
FatalError("A batch size greater than nBatches cannot be used");
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT feature extraction ", '=', 30);
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi){
if(!frames[bi].data)
FatalError("No image data feed to extract features");
preprocess(frames[bi], bi);
}
TKDNN_TSTOP
}
//do inference
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = cur_batches;
{
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
}
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi)
postprocess(bi);
TKDNN_TSTOP
}
}
/**
* Method to draw the result.
*
*/
void draw() { }
};
}}
#endif /* DEPTHNN_H*/
include/tkDNN/DetectionNN.h
+7 -9
View File
@@ -4,10 +4,7 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
@@ -17,7 +14,7 @@
#include "tkdnn.h"
//#define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
#ifdef OPENCV_CUDACONTRIB
#include <opencv2/cudawarping.hpp>
@@ -79,15 +76,15 @@ class DetectionNN {
~DetectionNN(){};
/**
* Method used to initialize the class, allocate memory and compute
* Method used to inialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file of the NN.
* @param tensor_path path to the rt file og the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference
* @return true if everything is correct, false otherwise.
*/
virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0;
virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1) = 0;
/**
* This method performs the whole detection of the NN.
@@ -144,15 +141,16 @@ class DetectionNN {
}
/**
* Method to draw bounding boxes and labels on a frame.
* Method to draw boundixg boxes and labels on a frame.
*
* @param frames original frame to draw bounding box on.
* @param frames orginal frame to draw bounding box on.
*/
void draw(std::vector<cv::Mat>& frames) {
tk::dnn::box b;
int x0, w, x1, y0, h, y1;
int objClass;
std::string det_class;
int baseline = 0;
float font_scale = 0.5;
int thickness = 2;
include/tkDNN/DetectionNN3D.h
-161
View File
@@ -1,161 +0,0 @@
#ifndef DETECTIONNN3D_H
#define DETECTIONNN3D_H
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "tkdnn.h"
// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
#ifdef OPENCV_CUDACONTRIB
#include <opencv2/cudawarping.hpp>
#include <opencv2/cudaarithm.hpp>
#endif
namespace tk { namespace dnn {
class DetectionNN3D {
protected:
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_d;
std::vector<cv::Size> originalSize;
cv::Scalar colors[256];
int nBatches = 1;
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat bgr[3];
cv::cuda::GpuMat imagePreproc;
#else
cv::Mat bgr[3];
cv::Mat imagePreproc;
dnnType *input;
#endif
/**
* This method preprocess the image, before feeding it to the NN.
*
* @param frame original frame to adapt for inference.
* @param bi batch index
*/
virtual void preprocess(cv::Mat &frame, const int bi=0) = 0;
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param bi batch index
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
virtual void postprocess(const int bi=0,const bool mAP=false) = 0;
public:
int classes = 0;
float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
std::vector<tk::dnn::box3D> detected3D; /*bounding boxes in output*/
std::vector<std::vector<tk::dnn::box3D>> batchDetected; /*bounding boxes in output*/
std::vector<double> pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/
std::vector<std::string> classesNames;
DetectionNN3D() {};
~DetectionNN3D(){};
/**
* Method used to initialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file of the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference.
* @return true if everything is correct, false otherwise.
*/
virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1,
const float conf_thresh=0.3, const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>()) = 0;
/**
* This method performs the whole detection of the NN.
*
* @param frames frames to run detection on.
* @param cur_batches number of batches to use in inference.
* @param save_times if set to true, preprocess, inference and postprocess times
* are saved on a csv file, otherwise not.
* @param times pointer to the output stream where to write times.
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation.
*/
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool save_times=false,
std::ofstream *times=nullptr, const bool mAP=false){
if(save_times && times==nullptr)
FatalError("save_times set to true, but no valid ofstream given");
if(cur_batches > nBatches)
FatalError("A batch size greater than nBatches cannot be used");
originalSize.clear();
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi){
if(!frames[bi].data)
FatalError("No image data feed to detection");
originalSize.push_back(frames[bi].size());
preprocess(frames[bi], bi);
}
TKDNN_TSTOP
pre_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
//do inference
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = cur_batches;
{
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
batchDetected.clear();
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi)
postprocess(bi, mAP);
TKDNN_TSTOP
post_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<"\n";
}
}
/**
* Method to draw bounding boxes and labels on a frame.
*
* @param frames original frame to draw bounding box on.
*/
virtual void draw(std::vector<cv::Mat>& frames){};
};
}}
#endif /* DETECTIONNN3D_H*/
include/tkDNN/ImuOdom.h
+2 -9
View File
@@ -1,14 +1,7 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#ifdef __linux__
#include <unistd.h>
#elif _WIN32
#define _USE_MATH_DEFINES
#include <math.h>
#endif
#include <mutex>
#include <Eigen/Dense>
#include "utils.h"
@@ -51,7 +44,7 @@ class ImuOdom {
virtual ~ImuOdom() {}
/**
* Method used for initialize the class
* Method used for inizialize the class
*
* @return Success of the initialization
*/
@@ -148,7 +141,7 @@ class ImuOdom {
//odomPOS = odomPOS + deltaP.cast<double>(); // V2
odomROT = odomROT * q.normalized().toRotationMatrix();
// compute Euler
// compute euler
auto newEULER = odomROT.eulerAngles(0, 1, 2);
for(int i=0; i<3; i++) {
while( fabs(newEULER(i) - odomEULER(i)) > M_PI_2 ) {
include/tkDNN/Int8BatchStream.h
+3 -6
View File
@@ -11,11 +11,8 @@
#include <fstream>
#include <iomanip>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <stdlib.h>
#include <unistd.h>
#endif
#include <mutex>
#include "NvInfer.h"
@@ -39,7 +36,7 @@ public:
float *getLabels() { return mLabels.data(); }
int getBatchesRead() const { return mBatchCount; }
int getBatchSize() const { return mBatchSize; }
nvinfer1::Dims4 getDims() const { return mDims; }
nvinfer1::DimsNCHW getDims() const { return mDims; }
float* getFileBatch() { return &mFileBatch[0]; }
float* getFileLabels() { return &mFileLabels[0]; }
void readInListFile(const std::string& dataFilePath, std::vector<std::string>& mListIn);
@@ -55,7 +52,7 @@ private:
int mFileBatchPos{ 0 };
int mImageSize{ 0 };
nvinfer1::Dims4 mDims;
nvinfer1::DimsNCHW mDims;
std::vector<float> mBatch;
std::vector<float> mLabels;
std::vector<float> mFileBatch;
include/tkDNN/Int8Calibrator.h
+4 -4
View File
@@ -30,10 +30,10 @@ public:
Int8EntropyCalibrator(BatchStream& stream, int firstBatch, const std::string& calibTableFilePath,
const std::string& inputBlobName, bool readCache = true);
virtual ~Int8EntropyCalibrator() { checkCuda(cudaFree(mDeviceInput)); }
int getBatchSize() const NOEXCEPT override { return mStream.getBatchSize(); }
bool getBatch(void* bindings[], const char* names[], int nbBindings) NOEXCEPT override;
const void* readCalibrationCache(size_t& length) NOEXCEPT override;
void writeCalibrationCache(const void* cache, size_t length) NOEXCEPT override;
int getBatchSize() const override { return mStream.getBatchSize(); }
bool getBatch(void* bindings[], const char* names[], int nbBindings) override;
const void* readCalibrationCache(size_t& length) override;
void writeCalibrationCache(const void* cache, size_t length) override;
private:
BatchStream mStream;
include/tkDNN/Layer.h
+20 -99
View File
@@ -19,10 +19,8 @@ enum layerType_t {
LAYER_ACTIVATION_CRELU,
LAYER_ACTIVATION_LEAKY,
LAYER_ACTIVATION_MISH,
LAYER_ACTIVATION_LOGISTIC,
LAYER_FLATTEN,
LAYER_RESHAPE,
LAYER_RESIZE,
LAYER_MULADD,
LAYER_POOLING,
LAYER_SOFTMAX,
@@ -31,8 +29,7 @@ enum layerType_t {
LAYER_SHORTCUT,
LAYER_UPSAMPLE,
LAYER_REGION,
LAYER_YOLO,
LAYER_PADDING,
LAYER_YOLO
};
#define TKDNN_BN_MIN_EPSILON 1e-5
@@ -57,10 +54,6 @@ public:
int id = 0;
bool final; //if the layer is the final one
unsigned int n_params = 0;
unsigned int feature_map_size = 0;
long unsigned MACC = 0;
std::string getLayerName() {
layerType_t type = getLayerType();
@@ -75,10 +68,8 @@ public:
case LAYER_ACTIVATION_CRELU: return "ActivationCReLU";
case LAYER_ACTIVATION_LEAKY: return "ActivationLeaky";
case LAYER_ACTIVATION_MISH: return "ActivationMish";
case LAYER_ACTIVATION_LOGISTIC: return "ActivationLogistic";
case LAYER_FLATTEN: return "Flatten";
case LAYER_RESHAPE: return "Reshape";
case LAYER_RESIZE: return "Resize";
case LAYER_MULADD: return "MulAdd";
case LAYER_POOLING: return "Pooling";
case LAYER_SOFTMAX: return "Softmax";
@@ -88,7 +79,6 @@ public:
case LAYER_UPSAMPLE: return "Upsample";
case LAYER_REGION: return "Region";
case LAYER_YOLO: return "Yolo";
case LAYER_PADDING: return "Padding";
default: return "unknown";
}
}
@@ -181,7 +171,7 @@ public:
/**
Input layer (it doesn't need weights)
Input layer (it doesnt need weigths)
*/
class Input : public Layer {
@@ -217,26 +207,24 @@ public:
/**
Available activation functions
Avaible activation functions
*/
typedef enum {
ACTIVATION_ELU = 100,
ACTIVATION_LEAKY = 101,
ACTIVATION_MISH = 102,
ACTIVATION_LOGISTIC = 103
ACTIVATION_MISH = 102
} tkdnnActivationMode_t;
/**
Activation layer (it doesn't need weights)
Activation layer (it doesnt need weigths)
*/
class Activation : public Layer {
public:
int act_mode;
float ceiling;
float slope;
Activation(Network *net, int act_mode, const float ceiling=0.0, const float slope=0.1);
Activation(Network *net, int act_mode, const float ceiling=0.0);
virtual ~Activation();
virtual layerType_t getLayerType() {
if(act_mode == CUDNN_ACTIVATION_CLIPPED_RELU)
@@ -245,8 +233,6 @@ public:
return LAYER_ACTIVATION_LEAKY;
else if (act_mode == ACTIVATION_MISH)
return LAYER_ACTIVATION_MISH;
else if (act_mode == ACTIVATION_LOGISTIC)
return LAYER_ACTIVATION_LOGISTIC;
else
return LAYER_ACTIVATION;
};
@@ -332,9 +318,9 @@ public:
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
const bool bidirectional = true; /**> is the net bidir */
bool returnSeq = false; /**> if false return only the result of last timestamp */
bool returnSeq = false; /**> if false return only the result of last timestep */
int stateSize = 0; /**> number of hidden states */
int seqLen = 0; /**> number of timestamp */
int seqLen = 0; /**> number of timesteps */
int numLayers = 1; /**> number of internal layers */
protected:
@@ -381,7 +367,7 @@ public:
/**
Deformable Convolutional 2d layer
Deformable Convolutionl 2d layer
*/
class DeformConv2d : public LayerWgs {
@@ -425,8 +411,6 @@ public:
virtual layerType_t getLayerType() { return LAYER_FLATTEN; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
int c, h, w, rows, cols;
};
/**
@@ -440,27 +424,9 @@ public:
virtual layerType_t getLayerType() { return LAYER_RESHAPE; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
int n,c,h,w;
};
enum ResizeMode_t { NEAREST= 0,
LINEAR= 1};
/**
Resize layer
*/
class Resize : public Layer {
public:
Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed=false, ResizeMode_t mode=NEAREST);
virtual ~Resize();
virtual layerType_t getLayerType() { return LAYER_RESIZE; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
ResizeMode_t mode;
};
/**
MulAdd layer
@@ -475,6 +441,7 @@ public:
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
protected:
dnnType mul, add;
dnnType *add_vector;
};
@@ -482,7 +449,7 @@ public:
/**
Available pooling functions (padding on tkDNN is not supported)
Avaible pooling functions (padding on tkDNN is not supported)
*/
typedef enum {
POOLING_MAX = 0,
@@ -493,7 +460,7 @@ typedef enum {
/**
Pooling layer
currently supported only 2d pooing (also on 3d input)
currenty supported only 2d pooing (also on 3d input)
*/
class Pooling : public Layer {
@@ -501,7 +468,6 @@ public:
int winH, winW;
int strideH, strideW;
int paddingH, paddingW;
int padding;
bool size;
tkdnnPoolingMode_t pool_mode;
@@ -521,35 +487,9 @@ protected:
bool poolOn3d;
};
/**
* Padding Layers
* tkDNN supports reflection,constant and zero padding
*/
typedef enum {
PADDING_MODE_CONSTANT = 0,
PADDING_MODE_ZERO = 1,
PADDING_MODE_REFLECTION = 2
} tkdnnPaddingMode_t;
class Padding : public Layer {
public:
Padding(Network *net,int32_t pad_h,int32_t pad_w,tkdnnPaddingMode_t padding_mode,float constant = 0.0);
virtual ~Padding();
virtual layerType_t getLayerType(){return LAYER_PADDING ;};
virtual dnnType* infer(dataDim_t& dim,dnnType* srcData);
int32_t paddingH,paddingW;
tkdnnPaddingMode_t padding_mode;
float constant;
};
/**
Softmax layer
*/
class Softmax : public Layer {
public:
@@ -586,7 +526,7 @@ public:
/**
Reorg layer
Maintains same dimension but change C*H*W distribution
Mantain same dimension but change C*H*W distribution
*/
class Reorg : public Layer {
@@ -607,22 +547,19 @@ public:
class Shortcut : public Layer {
public:
Shortcut(Network *net, Layer *backLayer, bool mul=false);
Shortcut(Network *net, Layer *backLayer);
virtual ~Shortcut();
virtual layerType_t getLayerType() { return LAYER_SHORTCUT; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
int c,h,w;
public:
Layer *backLayer;
bool mul = false;
};
/**
Upsample layer
Maintains same dimension but change C*H*W distribution
Mantain same dimension but change C*H*W distribution
*/
class Upsample : public Layer {
@@ -635,7 +572,6 @@ public:
int stride;
bool reverse;
int c,h,w;
};
struct box {
@@ -654,16 +590,6 @@ struct sortable_bbox {
int cl;
float **probs;
};
struct box3D {
int cl;
std::vector<float> corners;
float prob;
void print()
{
std::cout<<"\tcl: "<<cl<<"\tprob: "<<prob<<"\tshape corners: "<<corners.size()<<std::endl;
}
};
/**
Yolo3 layer
@@ -684,28 +610,24 @@ public:
int sort_class;
};
enum nmsKind_t {GREEDY_NMS=0, DIOU_NMS=1};
Yolo(Network *net, int classes, int num, std::string fname_weights,int n_masks=3, float scale_xy=1, double nms_thresh=0.45, nmsKind_t nsm_kind=GREEDY_NMS, int new_coords=0);
Yolo(Network *net, int classes, int num, std::string fname_weights,int n_masks=3, float scale_xy=1);
virtual ~Yolo();
virtual layerType_t getLayerType() { return LAYER_YOLO; };
int classes, num, n_masks, new_coords;
int classes, num, n_masks;
dnnType *mask_h, *mask_d; //anchors
dnnType *bias_h, *bias_d; //anchors
float scaleXY;
double nms_thresh;
nmsKind_t nsm_kind;
std::vector<std::string> classesNames;
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh, int new_coords=0);
int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh);
dnnType *predictions;
static const int MAX_DETECTIONS = 8192*2;
static const int MAX_DETECTIONS = 8192;
static Yolo::detection *allocateDetections(int nboxes, int classes);
static void mergeDetections(Yolo::detection *dets, int ndets, int classes, double nms_thresh=0.45, nmsKind_t nsm_kind=GREEDY_NMS);
static void mergeDetections(Yolo::detection *dets, int ndets, int classes);
};
/**
@@ -719,7 +641,6 @@ public:
virtual layerType_t getLayerType() { return LAYER_REGION; };
int classes, coords, num;
int c,h,w;
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
};
include/tkDNN/MobilenetDetection.h
+1 -1
View File
@@ -65,7 +65,7 @@ public:
MobilenetDetection() {};
~MobilenetDetection() {};
bool init(const std::string& tensor_path,const int n_classes, const int n_batches=1, const float conf_thresh=0.3);
bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
};
include/tkDNN/Network.h
+4 -5
View File
@@ -7,12 +7,12 @@
namespace tk { namespace dnn {
/**
Data representation between layers
Data rapresentation beetween layers
n = batch size
c = channels
h = height (lines)
h = heigth (lines)
w = width (rows)
l = length (3rd dimension)
l = lenght (3rd dimension)
*/
struct dataDim_t {
@@ -43,14 +43,13 @@ public:
void releaseLayers();
/**
Do inference for every added layer
Do inferece for every added layer
*/
dnnType* infer(dataDim_t &dim, dnnType* data);
bool addLayer(Layer *l);
void print();
const char *getNetworkRTName(const char *network_name);
void adjustFeatureMapSizeWithShortcuts();
cudnnDataType_t dataType;
cudnnTensorFormat_t tensorFormat;
include/tkDNN/NetworkRT.h
+48 -42
View File
@@ -6,30 +6,49 @@
#include "Network.h"
#include "Layer.h"
#include "NvInfer.h"
#include <memory>
#include <tkDNN/kernels.h>
#include <pluginsRT/ActivationLeakyRT.h>
#include <pluginsRT/ActivationLogisticRT.h>
#include <pluginsRT/ActivationMishRT.h>
#include <pluginsRT/ActivationReLUCeilingRT.h>
#include <pluginsRT/DeformableConvRT.h>
#include <pluginsRT/FlattenConcatRT.h>
#include <pluginsRT/MaxPoolingFixedSizeRT.h>
#include <pluginsRT/RegionRT.h>
#include <pluginsRT/ReorgRT.h>
#include <pluginsRT/ReshapeRT.h>
#include <pluginsRT/ResizeLayerRT.h>
#include <pluginsRT/RouteRT.h>
#include <pluginsRT/ShortcutRT.h>
#include <pluginsRT/UpsampleRT.h>
#include <pluginsRT/YoloRT.h>
#include <pluginsRT/ConstantPaddingRT.h>
#include <pluginsRT/ReflectionPadding.h>
namespace tk { namespace dnn {
template<typename T> void writeBUF(char*& buffer, const T& val)
{
*reinterpret_cast<T*>(buffer) = val;
buffer += sizeof(T);
}
template<typename T> T readBUF(const char*& buffer)
{
T val = *reinterpret_cast<const T*>(buffer);
buffer += sizeof(T);
return val;
}
using namespace nvinfer1;
#include "pluginsRT/ActivationLeakyRT.h"
#include "pluginsRT/ActivationReLUCeilingRT.h"
#include "pluginsRT/ActivationMishRT.h"
#include "pluginsRT/ReorgRT.h"
#include "pluginsRT/RegionRT.h"
#include "pluginsRT/RouteRT.h"
#include "pluginsRT/ShortcutRT.h"
#include "pluginsRT/YoloRT.h"
#include "pluginsRT/UpsampleRT.h"
#include "pluginsRT/ResizeLayerRT.h"
#include "pluginsRT/DeformableConvRT.h"
#include "pluginsRT/FlattenConcatRT.h"
#include "pluginsRT/ReshapeRT.h"
#include "pluginsRT/MaxPoolingFixedSizeRT.h"
class PluginFactory : IPluginFactory
{
public:
YoloRT *yolos[16];
int n_yolos;
virtual IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength);
};
class NetworkRT {
public:
@@ -40,7 +59,6 @@ public:
#if NV_TENSORRT_MAJOR >= 6
nvinfer1::IBuilderConfig *configRT;
#endif
nvinfer1::ICudaEngine *engineRT;
nvinfer1::IExecutionContext *contextRT;
@@ -48,12 +66,12 @@ public:
void* buffersRT[MAX_BUFFERS_RT];
dataDim_t buffersDIM[MAX_BUFFERS_RT];
int buf_input_idx, buf_output_idx;
bool builderActive = false;
dataDim_t input_dim, output_dim;
dnnType *output;
cudaStream_t stream;
std::vector<nvinfer1::YoloRT*> yolo_plugins; // yolo layers in network
PluginFactory *pluginFactory;
NetworkRT(Network *net, const char *name);
virtual ~NetworkRT();
@@ -73,7 +91,7 @@ public:
}
/**
Do inference
Do inferece
*/
dnnType* infer(dataDim_t &dim, dnnType* data);
void enqueue(int batchSize = 1);
@@ -85,29 +103,17 @@ public:
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Flatten *l);
nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reshape *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Resize *l);
nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Region *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Flatten *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reshape *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, DeformConv2d *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input,Padding *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor* input,MulAdd *l);
#if NV_TENSORRT_MAJOR > 5 && NV_TENSORRT_MAJOR < 8
bool serialize(const char *filename);
#else
bool serialize(const char *filename,nvinfer1::IHostMemory *ptr);
#endif
bool deserialize(const char *filename);
void destroy();
};
}}
include/tkDNN/NetworkViz.h
+2 -2
View File
@@ -5,8 +5,8 @@
namespace tk { namespace dnn {
cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, int classes=19);
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=0);
cv::Mat vizFloat2colorMap(cv::Mat map);
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim);
cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000);
}}
include/tkDNN/SegmentationNN.h
-407
View File
@@ -1,407 +0,0 @@
#ifndef SEGMENTATIONNN_H
#define SEGMENTATIONNN_H
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/core/hal/interface.h>
#include "tkdnn.h"
#include "NetworkViz.h"
#include "kernelsThrust.h"
namespace tk { namespace dnn {
class SegmentationNN {
protected:
tk::dnn::NetworkRT *netRT = nullptr;
int nBatches = 1;
std::vector<cv::Size> originalSize;
cv::Mat bgr[3];
dnnType *input;
dnnType *input_d;
float* confidences_h;
float * tmpInputData_d;
float *tmpOutData_d;
float *tmpOutData_h;
float *mean_d, *stddev_d;
cublasHandle_t cublasHandle;
void computeBorders(const int or_width, const int or_height, int& top, int& bottom, int& left, int&right){
top = 0;
bottom = 0;
left = 0;
right = 0;
if(or_height != or_width){
if(or_height < or_width){
top = (or_width - or_height)/2;
bottom = or_width - top - or_height;
}
else{
left = (or_height - or_width)/2;
right = or_height - left - or_width;
}
}
}
/**
* This method preprocess the image, before feeding it to the NN.
*
* @param frame original frame to adapt for inference.
* @param bi batch index
*/
void preprocess(cv::Mat &frame, const int bi=0) {
originalSize[bi] = frame.size();
frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0);
int H = frame.rows;
int W = frame.cols;
cv::Mat frame_cropped;
int top, bottom, left, right;
computeBorders(W, H, top, bottom, left, right);
cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) );
tk::dnn::dataDim_t idim = netRT->input_dim;
resize(frame_cropped, frame_cropped, cv::Size(idim.w, idim.h));
cv::split(frame_cropped, bgr);
for (int i = 0; i < idim.c; i++){
int idx = i * frame_cropped.rows * frame_cropped.cols;
int ch = idim.c-1 -i;
memcpy((void *)&input[idx + idim.tot()*bi], (void *)bgr[ch].data, frame_cropped.rows * frame_cropped.cols * sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d+ idim.tot()*bi, input + idim.tot()*bi, idim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d);
}
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param bi batch index
*/
void postprocess(const int bi=0, bool appy_colormap = true) {
dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
dataDim_t odim = netRT->output_dim;
matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h);
maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w);
checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost));
dataDim_t vdim = odim;
vdim.c = 1;
cv::Mat colored;
if(appy_colormap)
colored = vizData2Mat(tmpOutData_h, vdim, netRT->input_dim.h, netRT->input_dim.w, 0, classes, classes);
else{
cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h);
colored_fp32.convertTo(colored, CV_8UC1);
}
int max_dim = (originalSize[bi].width > originalSize[bi].height) ? originalSize[bi].width : originalSize[bi].height;
resize(colored, colored, cv::Size(max_dim, max_dim));
int top, bottom, left, right;
computeBorders(originalSize[bi].width, originalSize[bi].height, top, bottom, left, right);
cv::Rect roi(left,top,originalSize[bi].width, originalSize[bi].height);
cv::Mat or_size (colored, roi);
segmented[bi] = or_size;
};
public:
int classes = 0;
std::vector<double> stats; /*keeps track of inference times (ms)*/
std::vector<double> stats_pre;
std::vector<double> stats_post;
std::vector<std::string> classesNames;
std::vector<cv::Mat> segmented;
SegmentationNN() {
checkERROR( cublasCreate(&cublasHandle) );
};
~SegmentationNN(){
checkERROR( cublasDestroy(cublasHandle) );
};
/**
* Method used to inialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file og the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference
* @return true if everything is correct, false otherwise.
*/
bool init(const std::string& tensor_path, const int n_classes=19, const int n_batches=1){
std::cout<<(tensor_path).c_str()<<"\n";
if(!fileExist(tensor_path.c_str()))
FatalError("This file do not exists" + tensor_path );
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str());
classes = n_classes;
nBatches = n_batches;
checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
dataDim_t odim = netRT->output_dim;
checkCuda(cudaMallocHost(&confidences_h, sizeof(float) * odim.tot()));
checkCuda(cudaMalloc(&tmpInputData_d, sizeof(float) * odim.tot()));
checkCuda(cudaMalloc(&tmpOutData_d, sizeof(float) * odim.w*odim.h));
checkCuda(cudaMallocHost(&tmpOutData_h, sizeof(float) * odim.w*odim.h));
segmented.resize(nBatches);
originalSize.resize(nBatches);
std::vector<float> mean = {0.485, 0.456, 0.406};
std::vector<float> stddev = {0.229, 0.224, 0.225};
checkCuda(cudaMalloc(&mean_d, sizeof(float) * mean.size()));
checkCuda(cudaMalloc(&stddev_d, sizeof(float) * stddev.size()));
checkCuda(cudaMemcpyAsync(mean_d, mean.data(), mean.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream));
checkCuda(cudaMemcpyAsync(stddev_d, stddev.data(), stddev.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream));
return true;
return true;
}
/**
* This method performs the whole detection of the NN.
*
* @param frames frames to run detection on.
* @param cur_batches number of batches to use in inference
* @param save_times if set to true, preprocess, inference and postprocess times
* are saved on a csv file, otherwise not.
* @param times pointer to the output stream where to write times
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool apply_colormap=true){
if(cur_batches > nBatches)
FatalError("A batch size greater than nBatches cannot be used");
originalSize.clear();
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi){
if(!frames[bi].data)
FatalError("No image data feed to detection");
originalSize.push_back(frames[bi].size());
preprocess(frames[bi], bi);
}
TKDNN_TSTOP
stats_pre.push_back(t_ns);
}
//do inference
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = cur_batches;
{
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
}
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi)
postprocess(bi, apply_colormap);
TKDNN_TSTOP
stats_post.push_back(t_ns);
}
}
void updateOriginal(cv::Mat frame, bool apply_colormap=true){
std::vector<cv::Mat> splitted_frames;
int H, W, net_H, net_W;
int top = 0, bottom = 0, left = 0, right = 0;
std::vector<std::pair<int,int>> pos;
{
TKDNN_TSTART
cv::Size original_size = frame.size();
frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0);
H = frame.rows;
W = frame.cols;
net_H = netRT->input_dim.h;
net_W = netRT->input_dim.w;
cv::Mat frame_cropped;
if( H <= net_H && W <= net_W ){ // smaller size wrt network
top = (net_H - H)/2;
bottom = net_H - H - top ;
left = (net_W - W)/2;
right = net_W - W - left ;
cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) );
splitted_frames.push_back(frame_cropped);
}
else{ //bigger size wrt network
if(H < net_H || W < net_W){
if(H < net_H){
top = (net_H - H)/2;
bottom = net_H - H - top ;
}
else{
left = (net_W - W)/2;
right = net_W - W - left ;
}
cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0));
}
for(int x=0; x+net_W<=W ;){
for(int y=0; y+net_H <=H ; ){
cv::Rect roi(x, y, net_W, net_H);
cv::Mat image_roi = frame(roi);
splitted_frames.push_back(image_roi);
pos.push_back(std::make_pair(x,y));
y += net_H;
if(y == H)
break;
if(y + net_H > H) y = H - net_H;
}
x += net_W;
if(x == W)
break;
if(x + net_W > W) x = W - net_W;
}
}
tk::dnn::dataDim_t idim = netRT->input_dim;
if(splitted_frames.size()> nBatches)
FatalError(std::to_string(splitted_frames.size()) + " min batches required");
for(int bi=0; bi<splitted_frames.size();++bi){
cv::split(splitted_frames[bi], bgr);
for (int i = 0; i < idim.c; i++){
int idx = i * splitted_frames[bi].rows * splitted_frames[bi].cols;
int ch = idim.c-1 -i;
memcpy((void *)&input[idx + idim.tot()*bi], (void *)bgr[ch].data, splitted_frames[bi].rows * splitted_frames[bi].cols * sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d+ idim.tot()*bi, input + idim.tot()*bi, idim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d);
}
TKDNN_TSTOP
stats_pre.push_back(t_ns);
}
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = splitted_frames.size();
{
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
}
dataDim_t odim = netRT->output_dim;
std::vector<cv::Mat> out_img;
{
TKDNN_TSTART
for(int bi=0; bi<splitted_frames.size();++bi){
dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h);
maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w);
checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost));
dataDim_t vdim = odim;
vdim.c = 1;
cv::Mat colored;
if(apply_colormap)
colored = vizData2Mat(tmpOutData_h, vdim, netRT->input_dim.h, netRT->input_dim.w, 0, classes, classes);
else{
cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h);
colored_fp32.convertTo(colored, CV_8UC1);
}
out_img.push_back(colored);
}
cv::Mat seg(frame.size(), out_img[0].type());
if(out_img.size() == 1)
{
cv::Rect roi(left, top, W, H);
seg = out_img[0](roi);
}
else{
int bi=0;
if(top == 0 && left == 0){
for(int i=0; i<out_img.size(); ++i){
cv::Mat roi_collage = seg(cv::Rect( pos[i].first ,pos[i].second,out_img[i].cols,out_img[i].rows));
out_img[i].copyTo(roi_collage);
}
}
else{
FatalError("Not handled case")
}
}
segmented[0] = seg;
TKDNN_TSTOP
stats_post.push_back(t_ns);
}
}
/**
* Method to draw boundixg boxes and labels on a frame.
*/
cv::Mat draw(const int cur_batches=1) {
for(int i=0; i<cur_batches; ++i){
cv::imshow("segmented", segmented[i]);
cv::resizeWindow("segmented", cv::Size(512,288));
cv::waitKey(1);
}
return segmented[0];
}
};
}}
#endif /* SEGMENTATIONNN_H*/
include/tkDNN/TrackingNN.h
-158
View File
@@ -1,158 +0,0 @@
#ifndef TRACKINGNN_H
#define TRACKINGNN_H
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "tkdnn.h"
// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
#ifdef OPENCV_CUDACONTRIB
#include <opencv2/cudawarping.hpp>
#include <opencv2/cudaarithm.hpp>
#endif
namespace tk { namespace dnn {
class TrackingNN {
protected:
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_d;
std::vector<cv::Size> originalSize;
cv::Scalar colors[256];
int nBatches = 1;
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat bgr[3];
cv::cuda::GpuMat imagePreproc;
#else
cv::Mat bgr[3];
cv::Mat imagePreproc;
dnnType *input;
#endif
/**
* This method preprocess the image, before feeding it to the NN.
*
* @param frame original frame to adapt for inference.
* @param bi batch index
*/
virtual void preprocess(cv::Mat &frame, const int bi=0) = 0;
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param bi batch index
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
virtual void postprocess(const int bi=0,const bool mAP=false) = 0;
public:
int classes = 0;
float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
std::vector<double> pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/
std::vector<std::string> classesNames;
TrackingNN() {};
~TrackingNN(){};
/**
* Method used to initialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file of the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference.
* @return true if everything is correct, false otherwise.
*/
virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1,
const float conf_thresh=0.3, const bool mode_3d=true, const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>()) = 0;
/**
* This method performs the whole detection and tracking of the NN.
*
* @param frames frames to run detection and trcking on.
* @param cur_batches number of batches to use in inference.
* @param save_times if set to true, preprocess, inference and postprocess times
* are saved on a csv file, otherwise not.
* @param times pointer to the output stream where to write times.
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation.
*/
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool save_times=false,
std::ofstream *times=nullptr, const bool mAP=false){
if(save_times && times==nullptr)
FatalError("save_times set to true, but no valid ofstream given");
if(cur_batches > nBatches)
FatalError("A batch size greater than nBatches cannot be used");
originalSize.clear();
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi){
if(!frames[bi].data)
FatalError("No image data feed to detection");
originalSize.push_back(frames[bi].size());
preprocess(frames[bi], bi);
}
TKDNN_TSTOP
pre_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
//do inference
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = cur_batches;
{
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi)
postprocess(bi, mAP);
TKDNN_TSTOP
post_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<"\n";
}
}
/**
* Method to draw bounding boxes and labels on a frame.
*
* @param frames original frame to draw bounding box on.
*/
virtual void draw(std::vector<cv::Mat>& frames){};
};
}}
#endif /* TRACKINGNN_H*/
include/tkDNN/Yolo3Detection.h
+5 -4
View File
@@ -4,15 +4,16 @@
#include "opencv2/opencv.hpp"
#include "DetectionNN.h"
#include "DarknetParser.h"
namespace tk { namespace dnn {
namespace tk { namespace dnn {
class Yolo3Detection : public DetectionNN
{
private:
int num = 0;
int nMasks = 0;
int nDets = 0;
bool letterbox = false;
tk::dnn::Yolo::detection *dets = nullptr;
tk::dnn::Yolo* yolo[3];
@@ -21,10 +22,10 @@ private:
cv::Mat bgr_h;
public:
Yolo3Detection() {};
Yolo3Detection(const bool letter_box=false) :letterbox(letter_box){}
~Yolo3Detection() {};
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3);
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
};
include/tkDNN/demo_utils.h
-23
View File
@@ -1,23 +0,0 @@
#ifndef DEMO_UTILS_H
#define DEMO_UTILS_H
#include <iostream>
#include <sstream>
#include <fstream>
#include <iomanip>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <yaml-cpp/yaml.h>
void readCalibrationMatrix(const std::string& path, cv::Mat& calib_mat);
#endif //DEMO_UTILS_H
include/tkDNN/evaluation.h
+4 -6
View File
@@ -18,8 +18,6 @@ struct Frame
std::string iFilename;
std::vector<BoundingBox> gt;
std::vector<BoundingBox> det;
int width;
int height;
void print() const;
};
@@ -75,12 +73,12 @@ double computeMap( std::vector<Frame> &images,const int classes,
* all the recall levels are evaluated, otherwise only
* map_point recall levels are used. For COCO evaluation
* 101 points are used.
* @param map_step step used to increment IoU threshold
* @param map_step step used to increment IoU theshold
* @param map_levels number of IoU step to perform
* @param verbose is set to true, prints on screen additional info
* @param write_on_file if set to true, the results produced by this function
* are written on file
* @param net name of the considered neural network
* @param net name of the considerd neural network
*
* @return mAP IoU_tresh:IoU_tresh+map_step*map_levels (e.g. mAP 0.5:0.95 when
* map_step=0.05 and map_levels=10)
@@ -91,7 +89,7 @@ double computeMapNIoULevels(std::vector<Frame> &images,const int classes,
const int map_levels=10, const bool verbose=false,
const bool write_on_file = false, std::string net = "");
/**
* This method computes the number of True Positive (TP), False Positive (FP),
* This method computes the numper of True Positive (TP), False Positive (FP),
* False Negative (FN), precision, recall and f1-score.
* Those values are computer over all the detections, over all the classes.
*
@@ -103,7 +101,7 @@ double computeMapNIoULevels(std::vector<Frame> &images,const int classes,
* @param verbose is set to true, prints on screen additional info
* @param write_on_file if set to true, the results produced by this function
* are written on file
* @param net name of the considered neural network
* @param net name of the considerd neural network
*/
void computeTPFPFN( std::vector<Frame> &images,const int classes,
const float IoU_thresh=0.5, const float conf_thresh=0.3,
include/tkDNN/kernels.h
+2 -9
View File
@@ -4,7 +4,7 @@
#include "utils.h"
void activationELUForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
void activationLEAKYForward(dnnType *srcData, dnnType *dstData, int size, float slope, cudaStream_t stream = cudaStream_t(0));
void activationLEAKYForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
void activationReLUCeilingForward(dnnType *srcData, dnnType *dstData, int size, const float ceiling, cudaStream_t stream = cudaStream_t(0));
void activationLOGISTICForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
void activationSIGMOIDForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
@@ -24,7 +24,7 @@ void softmaxForward(float *input, int n, int batch, int batch_offset,
int groups, int group_offset, int stride, float temp, float *output, cudaStream_t stream = cudaStream_t(0));
void shortcutForward(dnnType *srcData, dnnType *dstData, int n1, int c1, int h1, int w1, int s1,
int n2, int c2, int h2, int w2, int s2, bool mul,
int n2, int c2, int h2, int w2, int s2,
cudaStream_t stream = cudaStream_t(0));
void upsampleForward(dnnType *srcData, dnnType *dstData,
@@ -48,11 +48,4 @@ void dcnV2CudaForward(cublasStatus_t stat, cublasHandle_t handle,
const int dst_dim, cudaStream_t stream = cudaStream_t(0));
void scalAdd(dnnType* dstData, int size, float alpha, float beta, int inc, cudaStream_t stream = cudaStream_t(0));
void reflection_pad2d_out_forward(int32_t pad_h,int32_t pad_w,float *srcData,float *dstData,int32_t input_h,int32_t input_w,int32_t plane_dim,int32_t n_batch,cudaStream_t cudaStream = cudaStream_t(0));
void constant_pad2d_forward(dnnType *srcData,dnnType *dstData,int32_t input_h,int32_t input_w,int32_t output_h,
int32_t output_w,int32_t c,int32_t n,int32_t padT,int32_t padL,dnnType constant,cudaStream_t cudaStream = cudaStream_t(0));
#endif //KERNELS_H
include/tkDNN/kernelsThrust.h
-7
View File
@@ -2,7 +2,6 @@
#define KERNELSTHRUST_H
#include <thrust/extrema.h>
#include <thrust/sort.h>
#include <thrust/execution_policy.h>
#include <thrust/functional.h>
@@ -10,8 +9,6 @@
#include <thrust/iterator/constant_iterator.h>
#include <thrust/gather.h>
#include <thrust/copy.h>
#include <thrust/device_ptr.h>
#include "tkdnn.h"
@@ -32,15 +29,11 @@ void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores,
int *topk_inds, float *topk_ys, float *topk_xs);
// void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes);
void normalize(float *bgr, const int ch, const int h, const int w, const float *mean, const float *stddev);
void transformDep(float *src_begin, float *src_end, float *dst_begin, float *dst_end);
void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op);
void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys);
void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin,
float *xs_begin, float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out);
void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin,
dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out);
void getRecordsFromTopKId(int * ids_begin, const int K, const int ch, const int size, dnnType *src_begin, float *src_out, int *ids_out);
void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w);
#endif //KERNELSTHRUST_H
include/tkDNN/pluginsRT/ActivationLeakyRT.h
+55 -83
View File
@@ -1,88 +1,60 @@
#include "NvInfer.h"
#include<cassert>
#include "../kernels.h"
#include <cassert>
#include <vector>
namespace nvinfer1 {
class ActivationLeakyRT : public IPluginV2 {
class ActivationLeakyRT : public IPlugin {
public:
explicit ActivationLeakyRT(float s);
ActivationLeakyRT(const void *data, size_t length);
~ActivationLeakyRT();
int getNbOutputs() const NOEXCEPT override;
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
void
configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,
PluginFormat format, int maxBatchSize) NOEXCEPT override;
int initialize() NOEXCEPT override;
void terminate() NOEXCEPT override {}
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, void const *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override;
void serialize(void *buffer) const NOEXCEPT override;
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
const char *getPluginType() const NOEXCEPT override;
const char *getPluginVersion() const NOEXCEPT override;
void destroy() NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
IPluginV2 *clone() const NOEXCEPT override;
int size;
float slope;
private:
std::string mPluginNamespace;
};
class ActivationLeakyRTPluginCreator : public IPluginCreator {
public:
ActivationLeakyRTPluginCreator();
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
public:
ActivationLeakyRT() {
REGISTER_TENSORRT_PLUGIN(ActivationLeakyRTPluginCreator);
};
}
~ActivationLeakyRT(){
}
int getNbOutputs() const override {
return 1;
}
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return inputs[0];
}
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
size = 1;
for(int i=0; i<outputDims[0].nbDims; i++)
size *= outputDims[0].d[i];
}
int initialize() override {
return 0;
}
virtual void terminate() override {
}
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
activationLEAKYForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
return 0;
}
virtual size_t getSerializationSize() override {
return 1*sizeof(int);
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, size);
}
int size;
};
include/tkDNN/pluginsRT/ActivationLogisticRT.h
-88
View File
@@ -1,88 +0,0 @@
#include<cassert>
#include "../kernels.h"
#include <NvInfer.h>
#include <vector>
#include <utils.h>
namespace nvinfer1 {
class ActivationLogisticRT : public IPluginV2 {
public:
ActivationLogisticRT() ;
ActivationLogisticRT(const void *data, size_t length) ;
~ActivationLogisticRT() ;
int getNbOutputs() const NOEXCEPT override ;
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,
PluginFormat format, int maxBatchSize) NOEXCEPT override ;
int initialize() NOEXCEPT override ;
void terminate() NOEXCEPT override ;
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override ;
void serialize(void *buffer) const NOEXCEPT override ;
const char *getPluginType() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
IPluginV2 *clone() const NOEXCEPT override ;
int size;
private:
std::string mPluginNamespace;
};
class ActivationLogisticRTPluginCreator : public IPluginCreator {
public:
ActivationLogisticRTPluginCreator() ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(ActivationLogisticRTPluginCreator);
};
include/tkDNN/pluginsRT/ActivationMishRT.h
+39 -61
View File
@@ -1,82 +1,60 @@
#include<cassert>
#include "../kernels.h"
#include <NvInfer.h>
#include <vector>
namespace nvinfer1 {
class ActivationMishRT : public IPluginV2 {
class ActivationMishRT : public IPlugin {
public:
ActivationMishRT() ;
~ActivationMishRT() ;
ActivationMishRT(const void *data, size_t length) ;
public:
ActivationMishRT() {
int getNbOutputs() const NOEXCEPT override ;
}
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
~ActivationMishRT(){
void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,
PluginFormat format, int maxBatchSize) NOEXCEPT override ;
}
int initialize() NOEXCEPT override ;
int getNbOutputs() const override {
return 1;
}
void terminate() NOEXCEPT override ;
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return inputs[0];
}
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
size = 1;
for(int i=0; i<outputDims[0].nbDims; i++)
size *= outputDims[0].d[i];
}
size_t getSerializationSize() const NOEXCEPT override ;
int initialize() override {
void serialize(void *buffer) const NOEXCEPT override ;
return 0;
}
const char *getPluginType() const NOEXCEPT override ;
virtual void terminate() override {
}
const char *getPluginVersion() const NOEXCEPT override ;
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
void destroy() NOEXCEPT override { delete this; }
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
activationMishForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
return 0;
}
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *plguinNamespace) NOEXCEPT override ;
virtual size_t getSerializationSize() override {
return 1*sizeof(int);
}
IPluginV2 *clone() const NOEXCEPT override ;
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, size);
}
int size;
private:
std::string mPluginNamespace;
};
class ActivationMishRTPluginCreator : public IPluginCreator {
public:
ActivationMishRTPluginCreator() ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(ActivationMishRTPluginCreator);
};
int size;
};
include/tkDNN/pluginsRT/ActivationReLUCeilingRT.h
+43 -62
View File
@@ -1,81 +1,62 @@
#include<cassert>
#include "../kernels.h"
#include <NvInfer.h>
#include <vector>
#include <utils.h>
namespace nvinfer1 {
class ActivationReLUCeiling : public IPluginV2 {
class ActivationReLUCeiling : public IPlugin {
public:
explicit ActivationReLUCeiling(const float ceiling) ;
public:
ActivationReLUCeiling(const float ceiling) {
this->ceiling = ceiling;
}
~ActivationReLUCeiling() ;
~ActivationReLUCeiling(){
ActivationReLUCeiling(const void *data, size_t length) ;
}
int getNbOutputs() const NOEXCEPT override ;
int getNbOutputs() const override {
return 1;
}
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return inputs[0];
}
void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,PluginFormat format, int maxBatchSize) NOEXCEPT override ;
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
size = 1;
for(int i=0; i<outputDims[0].nbDims; i++)
size *= outputDims[0].d[i];
}
int initialize() NOEXCEPT override ;
int initialize() override {
void terminate() NOEXCEPT override ;
return 0;
}
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
virtual void terminate() override {
}
size_t getSerializationSize() const NOEXCEPT override ;
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
void serialize(void *buffer) const NOEXCEPT override ;
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
IPluginV2 *clone() const NOEXCEPT override ;
activationReLUCeilingForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, ceiling, stream);
return 0;
}
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
virtual size_t getSerializationSize() override {
return 1*sizeof(int) + 1*sizeof(float);
}
const char *getPluginType() const NOEXCEPT override ;
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, ceiling);
tk::dnn::writeBUF(buf, size);
}
const char *getPluginVersion() const NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
int size;
float ceiling;
private:
std::string mPluginNamespace;
};
class ActivationReLUCeilingPluginCreator : public IPluginCreator {
public:
ActivationReLUCeilingPluginCreator() ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
public:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(ActivationReLUCeilingPluginCreator);
};
int size;
float ceiling;
};
include/tkDNN/pluginsRT/ActivationSigmoidRT.h
+1 -2
View File
@@ -52,9 +52,8 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, size);
assert(buf == a + getSerializationSize());
}
int size;
include/tkDNN/pluginsRT/ConstantPaddingRT.h
-109
View File
@@ -1,109 +0,0 @@
//
// Created by perseusdg on 1/7/22.
//
#ifndef _CONSTANTPADDINGRT_PLUGIN_H
#define _CONSTANTPADDINGRT_PLUGIN_H
#include<cassert>
#include <NvInfer.h>
#include <vector>
#include <utils.h>
#include <kernels.h>
namespace nvinfer1{
class ConstantPaddingRT : public IPluginV2Ext {
public:
ConstantPaddingRT(int32_t padH,int32_t padW,int32_t n,int32_t c,int32_t i_h,int32_t i_w,int32_t o_h,int32_t o_w,float constant);
ConstantPaddingRT(const void *data,size_t length);
~ConstantPaddingRT();
int getNbOutputs() const NOEXCEPT override;
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
int initialize() NOEXCEPT override ;
void terminate() NOEXCEPT override ;
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR <= 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override ;
void serialize(void *buffer) const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
const char *getPluginType() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
IPluginV2Ext *clone() const NOEXCEPT override ;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
bool supportsFormat (DataType type, PluginFormat format) const NOEXCEPT override;
int32_t i_h,i_w,o_h,o_w,n,c,padH,padW;
float constant;
private:
std::string mPluginNamespace;
};
class ConstantPaddingRTPluginCreator : public IPluginCreator {
public:
ConstantPaddingRTPluginCreator();
void setPluginNamespace(const char* pluginNamespace) NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(ConstantPaddingRTPluginCreator);
};
#endif //TKDNN_CONSTANTPADDINGRT_H
include/tkDNN/pluginsRT/DeformableConvRT.h
+183 -125
View File
@@ -1,137 +1,195 @@
#ifndef _DEFORMABLECONVRT_PLUGIN_H
#define _DEFORMABLECONVRT_PLUGIN_H
#include <NvInfer.h>
#include <vector>
#include<cassert>
#include "../kernels.h"
#include <tkdnn.h>
namespace nvinfer1 {
class DeformableConvRT : public IPluginV2Ext {
public:
DeformableConvRT(int chunk_dim, int kh, int kw, int sh, int sw, int ph, int pw,
int deformableGroup, int i_n, int i_c, int i_h, int i_w,
int o_n, int o_c, int o_h, int o_w,std::vector<dnnType> data_H,std::vector<dnnType> bias2_H,
std::vector<dnnType> ones_d1_h,std::vector<dnnType> ones_d2_h,std::vector<dnnType> offsetH,std::vector<dnnType> maskH,int height_ones,
int width_ones,int dim_ones);
~DeformableConvRT();
DeformableConvRT(const void *data, size_t length) ;
int getNbOutputs() const NOEXCEPT override ;
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
int initialize() NOEXCEPT override ;
void terminate() NOEXCEPT override ;
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override ;
void serialize(void *buffer) const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginType() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
IPluginV2Ext *clone() const NOEXCEPT override ;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
class DeformableConvRT : public IPlugin {
cublasStatus_t stat;
cublasHandle_t handle{nullptr};
int i_n, i_c, i_h, i_w;
int o_n, o_c, o_h, o_w;
int size;
int chunk_dim;
int kh, kw;
int sh, sw;
int ph, pw;
int deformableGroup;
int height_ones;
int width_ones;
int dim_ones;
std::vector<dnnType> data_d_v;
std::vector<dnnType> bias2_d_v;
std::vector<dnnType> ones_d1_v;
std::vector<dnnType> offset_v;
std::vector<dnnType> mask_v;
std::vector<dnnType> ones_d2_v;
dnnType* data_d;
dnnType* bias2_d;
dnnType* ones_d1;
dnnType* offset;
dnnType* mask;
dnnType* ones_d2;
// dnnType *input_n;
// dnnType *offset_n;
// dnnType *mask_n;
// dnnType *output_n;
public:
DeformableConvRT(int chunk_dim, int kh, int kw, int sh, int sw, int ph, int pw,
int deformableGroup, int i_n, int i_c, int i_h, int i_w,
int o_n, int o_c, int o_h, int o_w,
tk::dnn::DeformConv2d *deformable = nullptr) {
this->chunk_dim = chunk_dim;
this->kh = kh;
this->kw = kw;
this->sh = sh;
this->sw = sw;
this->ph = ph;
this->pw = pw;
this->deformableGroup = deformableGroup;
this->i_n = i_n;
this->i_c = i_c;
this->i_h = i_h;
this->i_w = i_w;
this->o_n = o_n;
this->o_c = o_c;
this->o_h = o_h;
this->o_w = o_w;
height_ones = (i_h + 2 * ph - (1 * (kh - 1) + 1)) / sh + 1;
width_ones = (i_w + 2 * pw - (1 * (kw - 1) + 1)) / sw + 1;
dim_ones = i_c * kh * kw * 1 * height_ones * width_ones;
std::cout<<i_c * o_c * kh * kw * 1<<"\n";
checkCuda( cudaMalloc(&data_d, i_c * o_c * kh * kw * 1 * sizeof(dnnType)));
checkCuda( cudaMalloc(&bias2_d, o_c*sizeof(dnnType)));
checkCuda( cudaMalloc(&ones_d1, height_ones * width_ones * sizeof(dnnType)));
checkCuda( cudaMalloc(&offset, 2*chunk_dim*sizeof(dnnType)));
checkCuda( cudaMalloc(&mask, chunk_dim*sizeof(dnnType)));
checkCuda( cudaMalloc(&ones_d2, dim_ones*sizeof(dnnType)));
if(deformable != nullptr) {
this->defRT = deformable;
checkCuda( cudaMemcpy(data_d, deformable->data_d, sizeof(dnnType)*i_c * o_c * kh * kw * 1, cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(bias2_d, deformable->bias2_d, sizeof(dnnType)*o_c, cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(ones_d1, deformable->ones_d1, sizeof(dnnType)*height_ones*width_ones, cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(offset, deformable->offset, sizeof(dnnType)*2*chunk_dim, cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(mask, deformable->mask, sizeof(dnnType)*chunk_dim, cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(ones_d2, deformable->ones_d2, sizeof(dnnType)*dim_ones, cudaMemcpyDeviceToDevice) );
}
stat = cublasCreate(&handle);
if (stat != CUBLAS_STATUS_SUCCESS)
FatalError("CUBLAS initialization failed\n");
}
~DeformableConvRT() {
checkCuda( cudaFree(data_d) );
checkCuda( cudaFree(bias2_d) );
checkCuda( cudaFree(ones_d1) );
checkCuda( cudaFree(offset) );
checkCuda( cudaFree(mask) );
checkCuda( cudaFree(ones_d2) );
cublasDestroy(handle);
}
int getNbOutputs() const override {
return 1;
}
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return DimsCHW{defRT->output_dim.c, defRT->output_dim.h, defRT->output_dim.w};
}
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override { }
int initialize() override {
return 0;
}
virtual void terminate() override { }
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
dnnType *output_conv = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
// split conv2d outputs into offset to mask
for(int b=0; b<batchSize; b++) {
checkCuda(cudaMemcpy(offset, output_conv + b * 3 * chunk_dim, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
checkCuda(cudaMemcpy(mask, output_conv + b * 3 * chunk_dim + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
// kernel sigmoide
activationSIGMOIDForward(mask, mask, chunk_dim);
// deformable convolution
dcnV2CudaForward(stat, handle,
srcData, data_d,
bias2_d, ones_d1,
offset, mask,
reinterpret_cast<dnnType*>(outputs[0]), ones_d2,
kh, kw,
sh, sw,
ph, pw,
1, 1,
deformableGroup, b,
i_n, i_c, i_h, i_w,
o_n, o_c, o_h, o_w,
chunk_dim);
}
return 0;
}
tk::dnn::DeformConv2d *defRT;
virtual size_t getSerializationSize() override {
return 16 * sizeof(int) + chunk_dim * 3 * sizeof(dnnType) + (i_c * o_c * kh * kw * 1 ) * sizeof(dnnType) +
o_c * sizeof(dnnType) + height_ones * width_ones * sizeof(dnnType) + dim_ones * sizeof(dnnType);
}
private:
std::string mPluginNamespace;
};
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, chunk_dim);
tk::dnn::writeBUF(buf, kh);
tk::dnn::writeBUF(buf, kw);
tk::dnn::writeBUF(buf, sh);
tk::dnn::writeBUF(buf, sw);
tk::dnn::writeBUF(buf, ph);
tk::dnn::writeBUF(buf, pw);
tk::dnn::writeBUF(buf, deformableGroup);
tk::dnn::writeBUF(buf, i_n);
tk::dnn::writeBUF(buf, i_c);
tk::dnn::writeBUF(buf, i_h);
tk::dnn::writeBUF(buf, i_w);
tk::dnn::writeBUF(buf, o_n);
tk::dnn::writeBUF(buf, o_c);
tk::dnn::writeBUF(buf, o_h);
tk::dnn::writeBUF(buf, o_w);
dnnType *aus = new dnnType[chunk_dim*2];
checkCuda( cudaMemcpy(aus, offset, sizeof(dnnType)*2*chunk_dim, cudaMemcpyDeviceToHost) );
for(int i=0; i<chunk_dim*2; i++)
tk::dnn::writeBUF(buf, aus[i]);
free(aus);
aus = new dnnType[chunk_dim];
checkCuda( cudaMemcpy(aus, mask, sizeof(dnnType)*chunk_dim, cudaMemcpyDeviceToHost) );
for(int i=0; i<chunk_dim; i++)
tk::dnn::writeBUF(buf, aus[i]);
free(aus);
aus = new dnnType[(i_c * o_c * kh * kw * 1 )];
checkCuda( cudaMemcpy(aus, data_d, sizeof(dnnType)*(i_c * o_c * kh * kw * 1 ), cudaMemcpyDeviceToHost) );
for(int i=0; i<(i_c * o_c * kh * kw * 1 ); i++)
tk::dnn::writeBUF(buf, aus[i]);
free(aus);
aus = new dnnType[o_c];
checkCuda( cudaMemcpy(aus, bias2_d, sizeof(dnnType)*o_c, cudaMemcpyDeviceToHost) );
for(int i=0; i < o_c; i++)
tk::dnn::writeBUF(buf, aus[i]);
free(aus);
aus = new dnnType[height_ones * width_ones];
checkCuda( cudaMemcpy(aus, ones_d1, sizeof(dnnType)*height_ones * width_ones, cudaMemcpyDeviceToHost) );
for(int i=0; i<height_ones * width_ones; i++)
tk::dnn::writeBUF(buf, aus[i]);
free(aus);
aus = new dnnType[dim_ones];
checkCuda( cudaMemcpy(aus, ones_d2, sizeof(dnnType)*dim_ones, cudaMemcpyDeviceToHost) );
for(int i=0; i<dim_ones; i++)
tk::dnn::writeBUF(buf, aus[i]);
free(aus);
}
class DeformableConvRTPluginCreator : public IPluginCreator {
public:
DeformableConvRTPluginCreator();
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(DeformableConvRTPluginCreator);
cublasStatus_t stat;
cublasHandle_t handle;
int i_n, i_c, i_h, i_w;
int o_n, o_c, o_h, o_w;
int size;
int chunk_dim;
int kh, kw;
int sh, sw;
int ph, pw;
int deformableGroup;
int height_ones;
int width_ones;
int dim_ones;
dnnType *data_d;
dnnType *bias2_d;
dnnType *ones_d1;
dnnType * offset;
dnnType * mask;
dnnType *ones_d2;
// dnnType *input_n;
// dnnType *offset_n;
// dnnType *mask_n;
// dnnType *output_n;
tk::dnn::DeformConv2d *defRT;
};
#endif
include/tkDNN/pluginsRT/FlattenConcatRT.h
+62 -82
View File
@@ -1,100 +1,80 @@
#ifndef _FLATTENCONCATRT_PLUGIN_H
#define _FLATTENCONCATRT_PLUGIN_H
#include<cassert>
#include <NvInfer.h>
#include <vector>
#include <utils.h>
namespace nvinfer1 {
class FlattenConcatRT : public IPluginV2Ext {
public:
FlattenConcatRT(int c,int h,int w,int rows,int cols) ;
class FlattenConcatRT : public IPlugin {
FlattenConcatRT(const void *data, size_t length) ;
public:
FlattenConcatRT() {
stat = cublasCreate(&handle);
if (stat != CUBLAS_STATUS_SUCCESS) {
printf ("CUBLAS initialization failed\n");
return;
}
}
~FlattenConcatRT() ;
~FlattenConcatRT(){
int getNbOutputs() const NOEXCEPT override ;
}
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
int getNbOutputs() const override {
return 1;
}
int initialize() NOEXCEPT override ;
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return DimsCHW{ inputs[0].d[0] * inputs[0].d[1] * inputs[0].d[2], 1, 1};
}
void terminate() NOEXCEPT override ;
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
assert(nbOutputs == 1 && nbInputs ==1);
rows = inputDims[0].d[0];
cols = inputDims[0].d[1] * inputDims[0].d[2];
c = inputDims[0].d[0] * inputDims[0].d[1] * inputDims[0].d[2];
h = 1;
w = 1;
}
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
int initialize() override {
return 0;
}
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR <= 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
virtual void terminate() override {
checkERROR(cublasDestroy(handle));
}
size_t getSerializationSize() const NOEXCEPT override ;
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
void serialize(void *buffer) const NOEXCEPT override ;
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*rows*cols*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
void destroy() NOEXCEPT override ;
checkERROR( cublasSetStream(handle, stream) );
for(int i=0; i<batchSize; i++) {
float const alpha(1.0);
float const beta(0.0);
int offset = i*rows*cols;
checkERROR( cublasSgeam( handle, CUBLAS_OP_T, CUBLAS_OP_N, rows, cols, &alpha, srcData + offset, cols, &beta, srcData + offset, rows, dstData + offset, rows ));
}
return 0;
}
const char *getPluginType() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override;
virtual size_t getSerializationSize() override {
return 5*sizeof(int);
}
const char *getPluginNamespace() const NOEXCEPT override ;
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
tk::dnn::writeBUF(buf, rows);
tk::dnn::writeBUF(buf, cols);
}
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
IPluginV2Ext *clone() const NOEXCEPT override ;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
bool supportsFormat (DataType type, PluginFormat format) const NOEXCEPT override;
int c, h, w;
int rows, cols;
cublasHandle_t handle{nullptr};
private:
std::string mPluginNamespace;
};
class FlattenConcatRTPluginCreator : public IPluginCreator {
public:
FlattenConcatRTPluginCreator() ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(FlattenConcatRTPluginCreator);
int c, h, w;
int rows, cols;
cublasStatus_t stat;
cublasHandle_t handle;
};
#endif
include/tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h
+66 -97
View File
@@ -1,105 +1,74 @@
#include<cassert>
#include "../kernels.h"
#include <NvInfer.h>
#include <vector>
#include <utils.h>
class MaxPoolFixedSizeRT : public IPlugin {
public:
MaxPoolFixedSizeRT(int c, int h, int w, int n, int strideH, int strideW, int winSize, int padding) {
this->c = c;
this->h = h;
this->w = w;
this->n = n;
this->stride_H = strideH;
this->stride_W = strideW;
this->winSize = winSize;
this->padding = padding;
}
~MaxPoolFixedSizeRT(){
}
int getNbOutputs() const override {
return 1;
}
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return DimsCHW{this->c, this->h, this->w};
}
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
}
int initialize() override {
return 0;
}
virtual void terminate() override {
}
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
//std::cout<<this->n<<" "<<this->c<<" "<<this->h<<" "<<this->w<<" "<<this->stride_H<<" "<<this->stride_W<<" "<<this->winSize<<" "<<this->padding<<std::endl;
dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
MaxPoolingForward(srcData, dstData, batchSize, this->c, this->h, this->w, this->stride_H, this->stride_W, this->winSize, this->padding, stream);
return 0;
}
namespace nvinfer1 {
class MaxPoolFixedSizeRT : public IPluginV2Ext {
virtual size_t getSerializationSize() override {
return 8*sizeof(int);
}
public:
MaxPoolFixedSizeRT(int c, int h, int w, int n, int strideH, int strideW, int winSize, int padding) ;
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
MaxPoolFixedSizeRT(const void *data, size_t length) ;
tk::dnn::writeBUF(buf, this->c);
tk::dnn::writeBUF(buf, this->h);
tk::dnn::writeBUF(buf, this->w);
tk::dnn::writeBUF(buf, this->n);
tk::dnn::writeBUF(buf, this->stride_H);
tk::dnn::writeBUF(buf, this->stride_W);
tk::dnn::writeBUF(buf, this->winSize);
tk::dnn::writeBUF(buf, this->padding);
}
~MaxPoolFixedSizeRT() ;
int getNbOutputs() const NOEXCEPT override ;
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
int initialize() NOEXCEPT override ;
void terminate() NOEXCEPT override ;
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR <= 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override ;
void serialize(void *buffer) const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginType() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
IPluginV2Ext *clone() const NOEXCEPT override ;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
int n, c, h, w;
int stride_H, stride_W;
int winSize;
int padding;
private:
std::string mPluginNamespace;
};
class MaxPoolFixedSizeRTPluginCreator : public IPluginCreator {
public:
MaxPoolFixedSizeRTPluginCreator() ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(MaxPoolFixedSizeRTPluginCreator);
int n, c, h, w;
int stride_H, stride_W;
int winSize;
int padding;
};
include/tkDNN/pluginsRT/ReflectionPadding.h
-101
View File
@@ -1,101 +0,0 @@
#ifndef _REFLECTIONPADDINGRT_PLUGIN_H
#define _REFLECTIONPADDINGRT_PLUGIN_H
#include<cassert>
#include <NvInfer.h>
#include <vector>
#include <utils.h>
#include <kernels.h>
namespace nvinfer1{
class ReflectionPaddingRT : public IPluginV2Ext {
public:
ReflectionPaddingRT(int32_t padH,int32_t padW,int32_t input_h,int32_t input_w,int32_t output_h,int32_t output_w,int32_t c,int32_t n);
ReflectionPaddingRT(const void *data,size_t length);
~ReflectionPaddingRT();
int getNbOutputs() const NOEXCEPT override;
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
int initialize() NOEXCEPT override ;
void terminate() NOEXCEPT override ;
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR <= 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override ;
void serialize(void *buffer) const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
const char *getPluginType() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
IPluginV2Ext *clone() const NOEXCEPT override ;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
bool supportsFormat (DataType type, PluginFormat format) const NOEXCEPT override;
int32_t padH,padW,input_h,input_w,output_h,output_w,n,c;
private:
std::string mPluginNamespace;
};
class ReflectionPaddingRTPluginCreator : public IPluginCreator {
public:
ReflectionPaddingRTPluginCreator();
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(ReflectionPaddingRTPluginCreator);
};
#endif
include/tkDNN/pluginsRT/RegionRT.h
+78 -94
View File
@@ -1,110 +1,94 @@
#ifndef _REGIONRT_PLUGIN_H
#define _REGIONRT_PLUGIN_H
#include<cassert>
#include "../kernels.h"
#include <NvInfer.h>
#include <vector>
#include <utils.h>
namespace nvinfer1 {
class RegionRT : public IPluginV2Ext {
class RegionRT : public IPlugin {
public:
RegionRT(int classes, int coords, int num,int c,int h,int w);
public:
RegionRT(int classes, int coords, int num) {
~RegionRT() ;
this->classes = classes;
this->coords = coords;
this->num = num;
}
RegionRT(const void *data, size_t length) ;
~RegionRT(){
int getNbOutputs() const NOEXCEPT override ;
}
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
int getNbOutputs() const override {
return 1;
}
int initialize() NOEXCEPT override ;
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return inputs[0];
}
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
c = inputDims[0].d[0];
h = inputDims[0].d[1];
w = inputDims[0].d[2];
}
int initialize() override {
return 0;
}
virtual void terminate() override {
}
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
for (int b = 0; b < batchSize; ++b){
for(int n = 0; n < num; ++n){
int index = entry_index(b, n*w*h, 0);
activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
index = entry_index(b, n*w*h, coords);
activationLOGISTICForward(srcData + index, dstData + index, w*h, stream);
}
}
//softmax start
int index = entry_index(0, 0, coords + 1);
softmaxForward( srcData + index, classes, batchSize*num,
(c*h*w)/num,
w*h, 1, w*h, 1, dstData + index, stream);
return 0;
}
void terminate() NOEXCEPT override ;
virtual size_t getSerializationSize() override {
return 6*sizeof(int);
}
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, classes);
tk::dnn::writeBUF(buf, coords);
tk::dnn::writeBUF(buf, num);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
}
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
int c, h, w;
int classes, coords, num;
int entry_index(int batch, int location, int entry) {
int n = location / (w*h);
int loc = location % (w*h);
return batch*c*h*w + n*w*h*(coords+classes+1) + entry*w*h + loc;
}
size_t getSerializationSize() const NOEXCEPT override ;
void serialize(void *buffer) const NOEXCEPT override ;
const char *getPluginType() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
IPluginV2Ext *clone() const NOEXCEPT override ;
int c, h, w;
int classes, coords, num;
int entry_index(int batch, int location, int entry) {
int n = location / (w * h);
int loc = location % (w * h);
return batch * c * h * w + n * w * h * (coords + classes + 1) + entry * w * h + loc;
}
private:
std::string mPluginNamespace;
};
class RegionRTPluginCreator : public IPluginCreator {
public:
RegionRTPluginCreator();
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(RegionRTPluginCreator);
};
#endif
include/tkDNN/pluginsRT/ReorgRT.h
+46 -81
View File
@@ -1,98 +1,63 @@
#include<cassert>
#include "../kernels.h"
#include <NvInfer.h>
#include <vector>
namespace nvinfer1 {
class ReorgRT : public IPluginV2Ext {
class ReorgRT : public IPlugin {
public:
ReorgRT(int stride,int c,int h,int w);
public:
ReorgRT(int stride) {
this->stride = stride;
}
~ReorgRT();
~ReorgRT(){
ReorgRT(const void *data, size_t length);
}
int getNbOutputs() const NOEXCEPT override;
int getNbOutputs() const override {
return 1;
}
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return DimsCHW{inputs[0].d[0]*stride*stride, inputs[0].d[1]/stride, inputs[0].d[2]/stride};
}
int initialize() NOEXCEPT override;
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
c = inputDims[0].d[0];
h = inputDims[0].d[1];
w = inputDims[0].d[2];
}
void terminate() NOEXCEPT override;
int initialize() override {
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
return 0;
}
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
virtual void terminate() override {
}
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
reorgForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
reinterpret_cast<dnnType*>(outputs[0]),
batchSize, c, h, w, stride, stream);
return 0;
}
size_t getSerializationSize() const NOEXCEPT override;
virtual size_t getSerializationSize() override {
return 4*sizeof(int);
}
void serialize(void *buffer) const NOEXCEPT override;
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, stride);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
}
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
const char *getPluginType() const NOEXCEPT override;
const char *getPluginVersion() const NOEXCEPT override;
void destroy() NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
IPluginV2Ext *clone() const NOEXCEPT override;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
int c, h, w, stride;
private:
std::string mPluginNamespace;
};
class ReorgRTPluginCreator : public IPluginCreator {
public:
ReorgRTPluginCreator();
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override;
const char *getPluginName() const NOEXCEPT override;
const char *getPluginVersion() const NOEXCEPT override;
const PluginFieldCollection *getFieldNames() NOEXCEPT override;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(ReorgRTPluginCreator);
int c, h, w, stride;
};
include/tkDNN/pluginsRT/ReshapeRT.h
+55 -95
View File
@@ -1,101 +1,61 @@
#ifndef _RESHAPERT_PLUGIN_H
#define _RESHAPERT_PLUGIN_H
#include<cassert>
#include <NvInfer.h>
#include <vector>
#include <tkdnn.h>
class ReshapeRT : public IPlugin {
public:
ReshapeRT(dataDim_t new_dim) {
n = new_dim.n;
c = new_dim.c;
h = new_dim.h;
w = new_dim.w;
}
~ReshapeRT(){
}
int getNbOutputs() const override {
return 1;
}
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return DimsCHW{ c,h,w};
}
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
}
int initialize() override {
return 0;
}
virtual void terminate() override {
}
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
return 0;
}
namespace nvinfer1 {
class ReshapeRT : public IPluginV2Ext {
virtual size_t getSerializationSize() override {
return 4*sizeof(int);
}
public:
ReshapeRT(int n,int c,int h,int w) ;
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, n);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
}
ReshapeRT(const void *data, size_t length) ;
~ReshapeRT() ;
int getNbOutputs() const NOEXCEPT override ;
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
int initialize() NOEXCEPT override ;
void terminate() NOEXCEPT override ;
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override ;
void serialize(void *buffer) const NOEXCEPT override ;
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
const char *getPluginType() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
IPluginV2Ext *clone() const NOEXCEPT override ;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
int n, c, h, w;
private:
std::string mPluginNamespace;
};
class ReshapeRTPluginCreator : public IPluginCreator {
public:
ReshapeRTPluginCreator() ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(ReshapeRTPluginCreator);
int n, c, h, w;
};
#endif
include/tkDNN/pluginsRT/ResizeLayerRT.h
+48 -85
View File
@@ -1,104 +1,67 @@
#include<cassert>
#include "../kernels.h"
#include <NvInfer.h>
#include <vector>
#include <utils.h>
namespace nvinfer1 {
class ResizeLayerRT : public IPlugin {
class ResizeLayerRT : public IPluginV2Ext {
public:
ResizeLayerRT(int c, int h, int w) {
o_c = c;
o_h = h;
o_w = w;
}
public:
ResizeLayerRT(int oc, int oh, int ow,int ic,int ih,int iw) ;
~ResizeLayerRT(){
}
ResizeLayerRT(const void *data, size_t length) ;
int getNbOutputs() const override {
return 1;
}
~ResizeLayerRT() ;
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return DimsCHW{o_c, o_h, o_w};
}
int getNbOutputs() const NOEXCEPT override ;
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
i_c = inputDims[0].d[0];
i_h = inputDims[0].d[1];
i_w = inputDims[0].d[2];
}
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
int initialize() override {
return 0;
}
int initialize() NOEXCEPT override ;
virtual void terminate() override {
}
void terminate() NOEXCEPT override ;
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR <= 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override ;
void serialize(void *buffer) const NOEXCEPT override ;
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
const char *getPluginType() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
IPluginV2Ext *clone() const NOEXCEPT override ;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
int i_c, i_h, i_w, o_c, o_h, o_w;
private:
std::string mPluginNamespace;
};
class ResizeLayerRTPluginCreator : public IPluginCreator {
public:
ResizeLayerRTPluginCreator() ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
// printf("%d %d %d %d %d %d\n", i_c, i_w, i_h, o_c, o_w, o_h);
resizeForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
reinterpret_cast<dnnType*>(outputs[0]),
batchSize, i_c, i_h, i_w, o_c, o_h, o_w, stream);
return 0;
}
virtual size_t getSerializationSize() override {
return 6*sizeof(int);
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
tk::dnn::writeBUF(buf, o_c);
tk::dnn::writeBUF(buf, o_h);
tk::dnn::writeBUF(buf, o_w);
};
tk::dnn::writeBUF(buf, i_c);
tk::dnn::writeBUF(buf, i_h);
tk::dnn::writeBUF(buf, i_w);
}
REGISTER_TENSORRT_PLUGIN(ResizeLayerRTPluginCreator);
int i_c, i_h, i_w, o_c, o_h, o_w;
};
include/tkDNN/pluginsRT/RouteRT.h
+72 -67
View File
@@ -1,90 +1,95 @@
#include<cassert>
#include "../kernels.h"
#include <vector>
#include <NvInfer.h>
namespace nvinfer1 {
class RouteRT : public IPluginV2 {
class RouteRT : public IPlugin {
/**
THIS IS NOT USED ANYMORE
*/
/**
THIS IS NOT USED ANYMORE
*/
public:
RouteRT(int groups, int group_id) ;
public:
RouteRT(int groups, int group_id) {
this->groups = groups;
this->group_id = group_id;
}
~RouteRT() ;
~RouteRT(){
RouteRT(const void *data, size_t length) ;
}
int getNbOutputs() const NOEXCEPT override ;
int getNbOutputs() const override {
return 1;
}
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
int out_c = 0;
for(int i=0; i<nbInputDims; i++) out_c += inputs[i].d[0];
return DimsCHW{out_c/groups, inputs[0].d[1], inputs[0].d[2]};
}
void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,PluginFormat format, int maxBatchSize) NOEXCEPT override ;
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
in = nbInputs;
c = 0;
for(int i=0; i<nbInputs; i++) {
c_in[i] = inputDims[i].d[0];
c += inputDims[i].d[0];
}
h = inputDims[0].d[1];
w = inputDims[0].d[2];
c /= groups;
}
int initialize() NOEXCEPT override ;
int initialize() override {
void terminate() NOEXCEPT override ;
return 0;
}
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
virtual void terminate() override {
}
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,cudaStream_t stream) NOEXCEPT override ;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
size_t getSerializationSize() const NOEXCEPT override ;
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
void serialize(void *buffer) const NOEXCEPT override ;
for(int b=0; b<batchSize; b++) {
int offset = 0;
for(int i=0; i<in; i++) {
dnnType *input = (dnnType*)reinterpret_cast<const dnnType*>(inputs[i]);
int in_dim = c_in[i]*h*w;
int part_in_dim = in_dim / this->groups;
checkCuda( cudaMemcpyAsync(dstData + b*c*w*h + offset, input + b*c*w*h*groups + this->group_id*part_in_dim, part_in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream) );
offset += part_in_dim;
}
}
const char *getPluginType() const NOEXCEPT override ;
return 0;
}
const char *getPluginVersion() const NOEXCEPT override ;
void destroy() NOEXCEPT override ;
virtual size_t getSerializationSize() override {
return (6+MAX_INPUTS)*sizeof(int);
}
const char *getPluginNamespace() const NOEXCEPT override ;
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, groups);
tk::dnn::writeBUF(buf, group_id);
tk::dnn::writeBUF(buf, in);
for(int i=0; i<MAX_INPUTS; i++)
tk::dnn::writeBUF(buf, c_in[i]);
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
}
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
IPluginV2 *clone() const NOEXCEPT override ;
static const int MAX_INPUTS = 4;
int in;
int c_in[MAX_INPUTS];
int c, h, w;
int groups, group_id;
private:
std::string mPluginNamespace;
};
class RouteRTPluginCreator : public IPluginCreator {
public:
RouteRTPluginCreator() ;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
const char *getPluginNamespace() const NOEXCEPT override ;
IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
const char *getPluginName() const NOEXCEPT override ;
const char *getPluginVersion() const NOEXCEPT override ;
const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(RouteRTPluginCreator);
static const int MAX_INPUTS = 4;
int in;
int c_in[MAX_INPUTS];
int c, h, w;
int groups, group_id;
};
include/tkDNN/pluginsRT/ShortcutRT.h
+67 -102
View File
@@ -1,109 +1,74 @@
#ifndef _SHORTCUTRT_PLUGIN_H
#define _SHORTCUTRT_PLUGIN_H
#include<cassert>
#include "../kernels.h"
#include <NvInfer.h>
#include <vector>
#include <tkdnn.h>
class ShortcutRT : public IPlugin {
public:
ShortcutRT(tk::dnn::dataDim_t bdim) {
this->bc = bdim.c;
this->bh = bdim.h;
this->bw = bdim.w;
}
~ShortcutRT(){
}
int getNbOutputs() const override {
return 1;
}
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return DimsCHW{inputs[0].d[0], inputs[0].d[1], inputs[0].d[2]};
}
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
c = inputDims[0].d[0];
h = inputDims[0].d[1];
w = inputDims[0].d[2];
}
int initialize() override {
return 0;
}
virtual void terminate() override {
}
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
dnnType *srcDataBack = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
for(int b=0; b < batchSize; ++b)
shortcutForward(srcDataBack + b*bc*bh*bw, dstData + b*c*h*w, 1, c, h, w, 1, 1, bc, bh, bw, 1, stream);
return 0;
}
namespace nvinfer1 {
virtual size_t getSerializationSize() override {
return 6*sizeof(int);
}
class ShortcutRT : public IPluginV2Ext {
public:
ShortcutRT(int bc,int bh,int bw,int c,int h,int w ,bool mul);
~ShortcutRT();
ShortcutRT(const void *data, size_t length);
int getNbOutputs() const NOEXCEPT override;
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims, int32_t nbOutputs,
DataType const *inputTypes, DataType const *outputTypes, bool const *inputIsBroadcast,
bool const *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch (int32_t outputIndex, bool const *inputIsBroadcasted, int32_t nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch (int32_t inputIndex) const NOEXCEPT override;
void attachToContext (cudnnContext *, cublasContext *, IGpuAllocator *) NOEXCEPT override;
void detachFromContext () NOEXCEPT override;
DataType getOutputDataType(int32_t index, nvinfer1::DataType const *inputTypes, int32_t nbInputs) const NOEXCEPT override;
int initialize() NOEXCEPT override;
void terminate() NOEXCEPT override;
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override;
void serialize(void *buffer) const NOEXCEPT override;
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
const char *getPluginType() const NOEXCEPT override;
const char *getPluginVersion() const NOEXCEPT override;
void destroy() NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
IPluginV2Ext *clone() const NOEXCEPT override;
int c, h, w;
int bc, bh, bw,bl;
bool mul;
tk::dnn::dataDim_t bDim;
private:
std::string mPluginNamespace;
};
class ShortcutRTPluginCreator : public IPluginCreator {
public:
ShortcutRTPluginCreator();
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override;
const char *getPluginName() const NOEXCEPT override;
const char *getPluginVersion() const NOEXCEPT override;
const PluginFieldCollection *getFieldNames() NOEXCEPT override;
public:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(ShortcutRTPluginCreator);
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, bc);
tk::dnn::writeBUF(buf, bh);
tk::dnn::writeBUF(buf, bw);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
}
int c, h, w;
int bc, bh, bw;
};
#endif
include/tkDNN/pluginsRT/UpsampleRT.h
+48 -86
View File
@@ -1,103 +1,65 @@
#ifndef _UPSAMPLERT_PLUGIN_H
#define _UPSAMPLERT_PLUGIN_H
#include<cassert>
#include "../kernels.h"
#include <NvInfer.h>
#include <vector>
namespace nvinfer1 {
class UpsampleRT : public IPlugin {
class UpsampleRT : public IPluginV2Ext {
public:
UpsampleRT(int stride) {
this->stride = stride;
}
public:
UpsampleRT(int stride,int c,int h,int w);
~UpsampleRT(){
UpsampleRT(const void *data, size_t length);
}
~UpsampleRT();
int getNbOutputs() const override {
return 1;
}
int getNbOutputs() const NOEXCEPT override;
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return DimsCHW(inputs[0].d[0], inputs[0].d[1]*stride, inputs[0].d[2]*stride);
}
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
c = inputDims[0].d[0];
h = inputDims[0].d[1];
w = inputDims[0].d[2];
}
int initialize() NOEXCEPT override;
int initialize() override {
void terminate() NOEXCEPT override;
return 0;
}
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
virtual void terminate() override {
}
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
fill(dstData, batchSize*c*h*w*stride*stride, 0.0, stream);
upsampleForward(srcData, dstData, batchSize, c, h, w, stride, 1, 1, stream);
return 0;
}
size_t getSerializationSize() const NOEXCEPT override;
virtual size_t getSerializationSize() override {
return 4*sizeof(int);
}
void serialize(void *buffer) const NOEXCEPT override;
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, stride);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
}
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
const char *getPluginType() const NOEXCEPT override;
const char *getPluginVersion() const NOEXCEPT override;
void destroy() NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
IPluginV2Ext *clone() const NOEXCEPT override ;
bool isOutputBroadcastAcrossBatch (int32_t outputIndex, bool const *inputIsBroadcasted, int32_t nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch (int32_t inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims, int32_t nbOutputs,
DataType const *inputTypes, DataType const *outputTypes, bool const *inputIsBroadcast,
bool const *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT override;
void attachToContext (cudnnContext *, cublasContext *, IGpuAllocator *) NOEXCEPT override;
void detachFromContext () NOEXCEPT override;
DataType getOutputDataType (int32_t index, nvinfer1::DataType const *inputTypes, int32_t nbInputs) const NOEXCEPT override;
int c, h, w, stride;
private:
std::string mPluginNamespace;
};
class UpsampleRTPluginCreator : public IPluginCreator {
public:
UpsampleRTPluginCreator();
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override;
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override;
const char *getPluginName() const NOEXCEPT override;
const char *getPluginVersion() const NOEXCEPT override;
const PluginFieldCollection *getFieldNames() NOEXCEPT override;
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
REGISTER_TENSORRT_PLUGIN(UpsampleRTPluginCreator);
};
#endif
int c, h, w, stride;
};
include/tkDNN/pluginsRT/YoloRT.h
+99 -101
View File
@@ -1,125 +1,123 @@
#ifndef _YOLORT_PLUGIN_H
#define _YOLORT_PLUGIN_H
#include<cassert>
#include <vector>
#include "../kernels.h"
#include <NvInfer.h>
#define YOLORT_CLASSNAME_W 256
namespace nvinfer1 {
class YoloRT : public IPluginV2Ext {
public:
YoloRT(int classes, int num,int c,int h,int w, int n_masks = 3, float scale_xy = 1,
float nms_thresh = 0.45, int nms_kind = 0, int new_coords = 0);
YoloRT(const void *data, size_t length);
~YoloRT();
class YoloRT : public IPlugin {
int getNbOutputs() const NOEXCEPT override;
Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
public:
YoloRT(int classes, int num, tk::dnn::Yolo *yolo = nullptr, int n_masks=3, float scale_xy=1) {
int initialize() NOEXCEPT override;
this->classes = classes;
this->num = num;
this->n_masks = n_masks;
this->scaleXY = scale_xy;
void terminate() NOEXCEPT override;
size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
#if NV_TENSORRT_MAJOR > 7
int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
cudaStream_t stream) NOEXCEPT override;
#elif NV_TENSORRT_MAJOR == 7
int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
#endif
size_t getSerializationSize() const NOEXCEPT override;
bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
void serialize(void *buffer) const NOEXCEPT override;
const char *getPluginType() const NOEXCEPT override;
const char *getPluginVersion() const NOEXCEPT override;
void destroy() NOEXCEPT override;
const char *getPluginNamespace() const NOEXCEPT override;
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
IPluginV2Ext *clone() const NOEXCEPT override;
DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
int32_t maxBatchSize) NOEXCEPT override;
void detachFromContext() NOEXCEPT override;
int c, h, w;
int classes, num, n_masks;
float scaleXY;
float nms_thresh;
int nms_kind;
int new_coords;
std::vector<std::string> classesNames;
std::vector<dnnType> mask;
std::vector<dnnType> bias;
int entry_index(int batch, int location, int entry) {
int n = location / (w * h);
int loc = location % (w * h);
return batch * c * h * w + n * w * h * (4 + classes + 1) + entry * w * h + loc;
mask = new dnnType[n_masks];
bias = new dnnType[num*n_masks*2];
if(yolo != nullptr) {
memcpy(mask, yolo->mask_h, sizeof(dnnType)*n_masks);
memcpy(bias, yolo->bias_h, sizeof(dnnType)*num*n_masks*2);
classesNames = yolo->classesNames;
}
}
private:
std::string mPluginNamespace;
~YoloRT(){
};
}
class YoloRTPluginCreator : public IPluginCreator {
public:
YoloRTPluginCreator();
int getNbOutputs() const override {
return 1;
}
void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return inputs[0];
}
const char *getPluginNamespace() const NOEXCEPT override;
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
c = inputDims[0].d[0];
h = inputDims[0].d[1];
w = inputDims[0].d[2];
}
IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
int initialize() override {
IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override;
return 0;
}
const char *getPluginName() const NOEXCEPT override;
virtual void terminate() override {
}
const char *getPluginVersion() const NOEXCEPT override;
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
const PluginFieldCollection *getFieldNames() NOEXCEPT override;
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
private:
static PluginFieldCollection mFC;
static std::vector<PluginField> mPluginAttributes;
std::string mPluginNamespace;
};
dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
for (int b = 0; b < batchSize; ++b){
for(int n = 0; n < n_masks; ++n){
int index = entry_index(b, n*w*h, 0);
activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
index = entry_index(b, n*w*h, 4);
activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream);
}
}
//std::cout<<"YOLO END\n";
return 0;
}
virtual size_t getSerializationSize() override {
return 6*sizeof(int) + sizeof(float)+ n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char);
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, classes);
tk::dnn::writeBUF(buf, num);
tk::dnn::writeBUF(buf, n_masks);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
tk::dnn::writeBUF(buf, scaleXY);
for(int i=0; i<n_masks; i++)
tk::dnn::writeBUF(buf, mask[i]);
for(int i=0; i<n_masks*2*num; i++)
tk::dnn::writeBUF(buf, bias[i]);
// save classes names
for(int i=0; i<classes; i++) {
char tmp[YOLORT_CLASSNAME_W];
strcpy(tmp, classesNames[i].c_str());
for(int j=0; j<YOLORT_CLASSNAME_W; j++) {
tk::dnn::writeBUF(buf, tmp[j]);
}
}
}
int c, h, w;
int classes, num, n_masks;
float scaleXY;
std::vector<std::string> classesNames;
dnnType *mask;
dnnType *bias;
int entry_index(int batch, int location, int entry) {
int n = location / (w*h);
int loc = location % (w*h);
return batch*c*h*w + n*w*h*(4+classes+1) + entry*w*h + loc;
}
REGISTER_TENSORRT_PLUGIN(YoloRTPluginCreator);
};
#endif
include/tkDNN/test.h
+4 -5
View File
@@ -20,7 +20,7 @@ int testInference(std::vector<std::string> input_bins, std::vector<std::string>
}
if(output_bins.size() != outputs.size()) {
std::cout<<output_bins.size()<<" "<<outputs.size()<<"\n";
FatalError("outputs size mismatch");
FatalError("outputs size missmatch");
}
// Load input
@@ -29,8 +29,7 @@ int testInference(std::vector<std::string> input_bins, std::vector<std::string>
readBinaryFile(input_bins[0], net->input_dim.tot(), &input_h, &data);
// outputs
//dnnType *cudnn_out[outputs.size()], *rt_out[outputs.size()];
std::vector<dnnType *> cudnn_out,rt_out;
dnnType *cudnn_out[outputs.size()], *rt_out[outputs.size()];
tk::dnn::dataDim_t dim1 = net->input_dim; //input dim
printCenteredTitle(" CUDNN inference ", '=', 30); {
@@ -40,7 +39,7 @@ int testInference(std::vector<std::string> input_bins, std::vector<std::string>
TKDNN_TSTOP
dim1.print();
}
for(int i=0; i<outputs.size(); i++) cudnn_out.push_back(outputs[i]->dstData);
for(int i=0; i<outputs.size(); i++) cudnn_out[i] = outputs[i]->dstData;
if(netRT != nullptr) {
tk::dnn::dataDim_t dim2 = net->input_dim;
@@ -51,7 +50,7 @@ int testInference(std::vector<std::string> input_bins, std::vector<std::string>
TKDNN_TSTOP
dim2.print();
}
for(int i=0; i<outputs.size(); i++) rt_out.push_back((dnnType*)netRT->buffersRT[i+1]);
for(int i=0; i<outputs.size(); i++) rt_out[i] = (dnnType*)netRT->buffersRT[i+1];
}
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
include/tkDNN/tkdnn.h
+1 -1
View File
@@ -5,4 +5,4 @@
#include "Layer.h"
#include "NetworkRT.h"
#define TKDNN_VERSION 700
#define TKDNN_VERSION 500
include/tkDNN/utils.h
+1 -58
View File
@@ -6,51 +6,18 @@
#include <fstream>
#include <iomanip>
#include <stdlib.h>
#include <yaml-cpp/yaml.h>
#include "cuda.h"
#include "cuda_runtime_api.h"
#include <cublas_v2.h>
#include <cudnn.h>
#include <NvInferVersion.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <ios>
#include <chrono>
#include <yaml-cpp/yaml.h>
#ifndef NOEXCEPT
#if NV_TENSORRT_MAJOR > 7
#define NOEXCEPT noexcept
#else
#define NOEXCEPT
#endif
#endif
#define dnnType float
template<typename T> void writeBUF(char*& buffer, const T& val)
{
*reinterpret_cast<T*>(buffer) = val;
buffer += sizeof(T);
}
template<typename T> T readBUF(const char*& buffer)
{
T val = *reinterpret_cast<const T*>(buffer);
buffer += sizeof(T);
return val;
}
// Colored output
#define COL_END "\033[0m"
@@ -72,7 +39,6 @@ template<typename T> T readBUF(const char*& buffer)
#define TKDNN_VERBOSE 0
// Simple Timer
#ifdef __linux__
#define TKDNN_TSTART timespec start, end; \
clock_gettime(CLOCK_MONOTONIC, &start);
@@ -82,14 +48,6 @@ template<typename T> T readBUF(const char*& buffer)
if(show) std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END;
#define TKDNN_TSTOP TKDNN_TSTOP_C(COL_CYANB, TKDNN_VERBOSE)
#elif _WIN32
#define TKDNN_TSTART auto start = std::chrono::high_resolution_clock::now();
#define TKDNN_TSTOP auto stop = std::chrono::high_resolution_clock::now(); \
std::chrono::duration<double> duration = stop -start; \
auto time_ms = std::chrono::duration_cast<std::chrono::milliseconds>(duration);\
double t_ns = time_ms.count();
#endif
/********************************************************
* Prints the error message, and exits
@@ -147,7 +105,7 @@ void printCenteredTitle(const char *title, char fill, int dim = 30);
bool fileExist(const char *fname);
void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string& test_folder, const std::string& weights_url);
void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0);
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10, bool verbose=true);
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10);
void printDeviceVector(int size, dnnType* vec_d, bool device = true);
float getColor(const int c, const int x, const int max);
void resize(int size, dnnType **data);
@@ -164,19 +122,4 @@ static inline bool isCudaPointer(void *data) {
cudaPointerAttributes attr;
return cudaPointerGetAttributes(&attr, data) == 0;
}
inline YAML::Node YAMLloadConf(const std::string& conf_file) {
std::cerr<<"Loading YAML: "<<conf_file<<"\n";
return YAML::LoadFile(conf_file);
}
template<typename T>
inline T YAMLgetConf(YAML::Node conf, std::string key, T defaultVal) {
T val = defaultVal;
if(conf && conf[key]) {
val = conf[key].as<T>();
}
return val;
}
#endif //UTILS_H
scripts/checkExecTimes.py
-37
View File
@@ -1,37 +0,0 @@
import sys
import pandas as pd
if len(sys.argv) < 3:
print("Error: two csv files are needed, old first new second")
exit(1)
old_perf_file = str(sys.argv[1])
new_perf_file = str(sys.argv[2])
verbose = False
if len(sys.argv) == 4:
verbose = bool(sys.argv[3])
print("Comparing {} vs {}".format(old_perf_file, new_perf_file))
df_old = pd.read_csv (old_perf_file, sep=';', header=None, index_col=0)
df_new = pd.read_csv (new_perf_file, sep=';', header=None, index_col=0)
for index, row in df_new.iterrows():
if index in df_old.index:
if verbose:
print("New: ",row[1], row[2], row[3])
print("Old: ",df_old.loc[index][1], df_old.loc[index][2], df_old.loc[index][3])
print(index, end=': ')
if abs(row[1] - df_old.loc[index][1]) < df_old.loc[index][1]*0.1:
print("similar performance")
elif (row[1] < df_old.loc[index][1]):
print('\x1b[3;30;42m' + 'faster' + '\x1b[0m')
elif (row[1] > df_old.loc[index][1]):
if row[1] > df_old.loc[index][1] + df_old.loc[index][1] * 0.5 :
print('\x1b[3;30;41m' + 'WAY SLOWER' + '\x1b[0m')
else:
print('\x1b[3;30;41m' + 'slower' + '\x1b[0m')
scripts/download_validation.py
-39
View File
@@ -1,39 +0,0 @@
import os
import urllib.request as dowReq
import zipfile
val = input("Enter BDD or COCO :")
if(val == "COCO"):
url = "https://cloud.hipert.unimore.it/s/LNxBDk4wzqXPL8c/download"
lib = "..\demo\COCO_val2017"
lib_zip = "COCO_val2017.zip"
elif(val == "BDD"):
url = "https://cloud.hipert.unimore.it/s/bikqk3FzCq2tg4D/download"
lib = "..\demo\BDD100k_val"
lib_zip = "BDD100k_val.zip"
dowReq.urlretrieve(url,lib_zip)
with zipfile.ZipFile(lib_zip,'r') as zip_ref:
zip_ref.extractall(lib)
labelFolder = lib + "\labels"
imageFolder = lib + "\images"
file1 = open(".\\..\\demo\\all_labels.txt","a")
path1 = os.path.realpath(labelFolder)
for file in os.listdir(labelFolder):
valTemp = path1 + "\\" + file
valTemp = valTemp + '\n'
file1.write(valTemp)
file1.close()
file2 = open(".\\..\\demo\\all_images.txt","a")
path2 = os.path.realpath(imageFolder)
for file in os.listdir(imageFolder):
pathtemp = path2 + "\\" + file
pathtemp = pathtemp + '\n'
file2.write(pathtemp)
file2.close()
print("Completed")
scripts/install_OpenCV4.sh
+2 -8
View File
@@ -27,21 +27,17 @@ sudo apt-get install -y build-essential \
libgstreamer1.0-dev \
libgstreamer-plugins-base1.0-dev \
libdc1394-22-dev \
libavresample-dev \
libtbb-dev \
libavresample-dev
git clone https://github.com/opencv/opencv.git
cd opencv && git checkout 4.5.4 && cd ..
git clone https://github.com/opencv/opencv_contrib.git
cd opencv_contrib && git checkout 4.5.4 && cd ..
python3 -m venv opencv4
source opencv4/bin/activate
pip install wheel
pip install numpy
cd opencv && mkdir build && cd build
cd opencv && mkdir build && cd build
cmake -D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=/usr/local \
@@ -60,8 +56,6 @@ cmake -D CMAKE_BUILD_TYPE=RELEASE \
-D WITH_GSTREAMER=ON \
-D WITH_GSTREAMER_0_10=OFF \
-D WITH_TBB=ON \
-D WITH_OPENGL=ON \
-D WITH_VULKAN=ON \
../
make -j4
scripts/test_all_tests.sh
+29 -45
View File
@@ -1,6 +1,6 @@
#!/bin/bash
#cd build
cd build
RED='\033[1;31m'
GREEN='\033[1;32m'
@@ -29,28 +29,24 @@ function print_output {
}
out_dir=results
out_file=results.log
rm -rf $out_dir/
mkdir -p $out_dir
rm $out_file
function test_net {
./test_$1 &> $out_dir/$1_${TKDNN_MODE}_build_$out_file
./test_$1 &>> $out_file
print_output $? $1
./test_rtinference $1*.rt 1 &> $out_dir/$1_${TKDNN_MODE}_inference_batch1_$out_file
print_output $? "infer $1"
./test_rtinference $1*.rt $TKDNN_BATCHSIZE &> $out_dir/$1_${TKDNN_MODE}_inference_batch${TKDNN_BATCHSIZE}_$out_file
./test_rtinference $1*.rt $TKDNN_BATCHSIZE &>> $out_file
print_output $? "batched $1"
}
# modes=( 1 ) # only FP32
modes=( 1 2 ) # FP32 and FP16
modes=( 1 ) # only FP32
# modes=( 1 2 ) # FP32 and FP16
# modes=( 1 2 3 ) # FP32, FP16 and INT8
for i in "${modes[@]}"
do
rm -f *rt
rm *rt
if [ $i -eq 1 ]
then
export TKDNN_MODE=FP32
@@ -73,41 +69,29 @@ do
echo -e "${ORANGE}Batch $TKDNN_BATCHSIZE ${NC}"
test_net mnist
# ./test_imuodom &>> $out_file
# print_output $? imuodom
./test_imuodom &>> $out_file
print_output $? imuodom
test_net yolo4
# test_net yolo4_320
# test_net yolo4_320_coco2
# test_net yolo4_512
# test_net yolo4_608
# test_net yolo4-csp
# test_net yolo4x
# test_net yolo4_berkeley
# test_net yolo4_berkeley_f1
# test_net yolo4tiny
# test_net yolo4tiny_512
# test_net yolo3
# test_net yolo3_berkeley
# test_net yolo3_coco4
# test_net yolo3_flir
# test_net yolo3_512
# test_net yolo3tiny
# test_net yolo3tiny_512
# test_net yolo2
# test_net yolo2_voc
# test_net yolo2tiny
# test_net csresnext50-panet-spp
# test_net csresnext50-panet-spp_berkeley
# test_net resnet101_cnet
# test_net dla34_cnet
# test_net dla34_cnet3d
# test_net mobilenetv2ssd
# test_net mobilenetv2ssd512
# test_net bdd-mobilenetv2ssd
# test_net dla34_ctrack
# test_net shelfnet
# test_net shelfnet_berkeley
test_net yolo4_berkeley
test_net yolo4tiny
test_net yolo3
test_net yolo3_berkeley
test_net yolo3_coco4
test_net yolo3_flir
test_net yolo3_512
test_net yolo3tiny
test_net yolo3tiny_512
test_net yolo2
test_net yolo2_voc
#test_net yolo2tiny
test_net csresnext50-panet-spp
#test_net csresnext50-panet-spp_berkeley
test_net resnet101_cnet
test_net dla34_cnet
test_net mobilenetv2ssd
test_net mobilenetv2ssd512
test_net bdd-mobilenetv2ssd
done
echo "If errors occured, check logfiles in directory: $out_dir"
echo "If errors occured, check logfile $out_file"
scripts/test_inference.sh
-52
View File
@@ -1,52 +0,0 @@
#!/bin/bash
function test_inference {
./test_$1
./test_rtinference $1_$2.rt 1
./test_rtinference $1_$2.rt 4
}
sudo jeston_clock
# modes=( 1 ) # only FP32
# modes=( 1 2 ) # FP32 and FP16
modes=( 1 2 3 ) # FP32, FP16 and INT8
rm times_rtinference.csv
for i in "${modes[@]}"
do
rm *rt
if [ $i -eq 1 ]
then
export TKDNN_MODE=FP32
mode=fp32
echo -e "${ORANGE}Test FP32${NC}"
fi
if [ $i -eq 2 ]
then
export TKDNN_MODE=FP16
mode=fp16
echo -e "${ORANGE}Test FP16${NC}"
fi
if [ $i -eq 3 ]
then
export TKDNN_MODE=INT8
export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
mode=int8
echo -e "${ORANGE}Test INT8${NC}"
fi
export TKDNN_BATCHSIZE=4
echo -e "${ORANGE}Batch $TKDNN_BATCHSIZE ${NC}"
test_inference yolo4_320 $mode
test_inference yolo4 $mode
test_inference yolo4_512 $mode
test_inference yolo4_608 $mode
test_inference yolo4tiny $mode
done
src/Activation.cpp
+5 -12
View File
@@ -5,12 +5,11 @@
namespace tk { namespace dnn {
Activation::Activation(Network *net, int act_mode, const float ceiling, const float slope) :
Activation::Activation(Network *net, int act_mode, const float ceiling) :
Layer(net) {
this->act_mode = act_mode;
this->ceiling = ceiling;
this->slope = slope;
this->act_mode = act_mode;
this->ceiling = ceiling;
checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
if(int(act_mode) < 100) {
@@ -47,18 +46,12 @@ Activation::~Activation() {
dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) {
if(act_mode == ACTIVATION_LEAKY) {
activationLEAKYForward(srcData, dstData, dim.tot(), this->slope);
activationLEAKYForward(srcData, dstData, dim.tot());
}
else if(act_mode == ACTIVATION_MISH) {
activationMishForward(srcData, dstData, dim.tot());
}
else if(act_mode == ACTIVATION_LOGISTIC) {
activationLOGISTICForward(srcData, dstData, dim.tot());
} else if(act_mode == ACTIVATION_ELU) {
activationELUForward(srcData, dstData, dim.tot());
} else {
dnnType alpha = dnnType(1);
dnnType beta = dnnType(0);
src/CenterTrack.cpp
-897
View File
@@ -1,897 +0,0 @@
#include "CenterTrack.h"
namespace tk { namespace dnn {
bool CenterTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches,
const float conf_thresh, const bool mode_3d, const std::vector<cv::Mat>& k_calibs) {
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
dim = netRT->input_dim;
dim.c = 3;
nBatches = n_batches;
confThreshold = conf_thresh;
mode3D = mode_3d;
inputCalibs = k_calibs;
init_preprocessing();
init_pre_inf();
init_postprocessing();
init_visualization(n_classes);
return true;
}
bool CenterTrack::init_preprocessing(){
//image transformation
src = cv::Mat(cv::Size(2,3), CV_32F);
dst = cv::Mat(cv::Size(2,3), CV_32F);
dst2 = cv::Mat(cv::Size(2,3), CV_32F);
trans = cv::Mat(cv::Size(3,2), CV_32F);
trans2 = cv::Mat(cv::Size(3,2), CV_32F);
transOut = cv::Mat(cv::Size(3,2), CV_32F);
dst2.at<float>(0,0) = width * 0.5;
dst2.at<float>(0,1) = width * 0.5;
dst2.at<float>(1,0) = width * 0.5;
dst2.at<float>(1,1) = width * 0.5 + width * -0.5;
dst2.at<float>(2,0) = dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
dst2.at<float>(2,1) = dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
for(int bi=0; bi<nBatches; bi++) {
szOld.push_back(cv::Size(0,0));
}
#ifdef OPENCV_CUDACONTRIB
std::cout<<"OPENCV CPMTROB\n";
checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) );
checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) );
float mean[3] = {0.40789655, 0.44719303, 0.47026116};
float stddev[3] = {0.2886383, 0.27408165, 0.27809834};
checkCuda( cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
checkCuda( cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
#else
std::cout<<"NO OPENCV CPMTROB\n";
checkCuda( cudaMallocHost(&input, sizeof(dnnType)*dim.tot() * nBatches));
mean << 0.40789655, 0.44719303, 0.47026116;
stddev << 0.2886383, 0.27408165, 0.27809834;
#endif
checkCuda( cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
checkCuda( cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot()));
checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) );
return true;
}
bool CenterTrack::init_pre_inf(){
// initial steps: the first part of the network
const char *pre_img_conv1_bin = "dla34_ctrack/layers/base-pre_img_layer-0.bin";
const char *pre_hm_conv1_bin = "dla34_ctrack/layers/base-pre_hm_layer-0.bin";
const char *conv1_bin = "dla34_ctrack/layers/base-base_layer-0.bin";
const char *conv2_bin = "dla34_ctrack/layers/base-level0-0.bin";
dim_in0 = tk::dnn::dataDim_t(1, 3, 512, 512, 1);
dim_in1 = tk::dnn::dataDim_t(1, 1, 512, 512, 1);
checkCuda( cudaMalloc(&out_d, netRT->input_dim.tot()*sizeof(dnnType)) );
checkCuda( cudaMalloc(&img_d, dim_in0.tot()*sizeof(dnnType)) );
checkCuda( cudaMalloc(&hm_d, dim_in1.tot()*sizeof(dnnType)) );
// init to zeros hm
dnnType *hm_h;
checkCuda( cudaMallocHost(&hm_h, 1 * dim.h * dim.w*sizeof(dnnType)) );
for(int i=0; i<1 * dim.h * dim.w; i++)
hm_h[i] = 0.0f;
checkCuda( cudaMemcpy(hm_d, hm_h, 1 * dim.h * dim.w * sizeof(dnnType), cudaMemcpyHostToDevice) );
checkCuda( cudaFreeHost(hm_h) );
dnnType *i0_h, *i1_h, *i2_h;
// dnnType *i0_d, *i1_d, *i2_d;
// const char *input_bin = "dla34_ctrack/debug/input.bin";
// const char *pre_img_bin = "dla34_ctrack/debug/pre_imgages.bin";
// const char *pre_hm_bin = "dla34_ctrack/debug/pre_hms.bin";
// readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &img_d);
// readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &hm_d);
// readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &input_pre_inf_d);
pre_phase_net = new tk::dnn::Network(dim_in0);
//pre-img
tk::dnn::Input *in_pre_img = new tk::dnn::Input(pre_phase_net, dim_in0, img_d);
tk::dnn::Conv2d *pre_img_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true);
tk::dnn::Activation *pre_img_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU);
//pre-hm
tk::dnn::Input *in_pre_hm = new tk::dnn::Input(pre_phase_net, dim_in1, hm_d);
tk::dnn::Conv2d *pre_hm_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true);
tk::dnn::Activation *pre_hm_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU);
// image input
tk::dnn::Input *input_image = new tk::dnn::Input(pre_phase_net, dim_in0, input_pre_inf_d);
tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true);
tk::dnn::Activation *relu1 = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU);
tk::dnn::Shortcut *s0_input = new tk::dnn::Shortcut(pre_phase_net, pre_img_relu);
tk::dnn::Shortcut *s1_input = new tk::dnn::Shortcut(pre_phase_net, pre_hm_relu);
// output data
out_d = s1_input->dstData;
//print network model
pre_phase_net->print();
iter0=true; // in the first iteration the last input is equal to the current input.
return true;
}
bool CenterTrack::init_postprocessing(){
srand(0); //seed = 0 for random colors
dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1);
dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_track = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1);
dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1);
dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
dim_amodel_offset = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
for(int i=0; i<dim_hm.c * dim_hm.h * dim_hm.w; i++){
ids_[i] = i;
}
checkCuda( cudaMalloc(&ones, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
float *ones_h;
checkCuda( cudaMallocHost(&ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
for(int i=0; i<dim_dep.c * dim_dep.h * dim_dep.w; i++)
ones_h[i] = 1.0f;
checkCuda( cudaMemcpy(ones, ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float), cudaMemcpyHostToDevice) );
checkCuda( cudaFreeHost(ones_h) );
checkCuda( cudaMallocHost(&scores, K *sizeof(float)) );
checkCuda( cudaMalloc(&scores_d, K *sizeof(float)) );
checkCuda( cudaMallocHost(&clses, K *sizeof(int)) );
checkCuda( cudaMalloc(&clses_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_inds_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_d, K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_d, K *sizeof(float)) );
checkCuda( cudaMalloc(&inttopk_ys_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&inttopk_xs_d, K *sizeof(int)) );
checkCuda( cudaMallocHost(&bbx0, K * sizeof(float)) );
checkCuda( cudaMallocHost(&bby0, K * sizeof(float)) );
checkCuda( cudaMallocHost(&bbx1, K * sizeof(float)) );
checkCuda( cudaMallocHost(&bby1, K * sizeof(float)) );
checkCuda( cudaMalloc(&bbx0_d, K * sizeof(float)) );
checkCuda( cudaMalloc(&bby0_d, K * sizeof(float)) );
checkCuda( cudaMalloc(&bbx1_d, K * sizeof(float)) );
checkCuda( cudaMalloc(&bby1_d, K * sizeof(float)) );
checkCuda( cudaMallocHost(&intxs, K * sizeof(int)) );
checkCuda( cudaMallocHost(&intys, K * sizeof(int)) );
checkCuda( cudaMallocHost(&track, K * dim_track.c * sizeof(float)) );
checkCuda( cudaMallocHost(&dep, K * dim_dep.c * sizeof(float)) );
checkCuda( cudaMallocHost(&rot, K * dim_rot.c * sizeof(float)) );
checkCuda( cudaMallocHost(&dim_, K * dim_dim.c * sizeof(float)) );
checkCuda( cudaMallocHost(&wh, K * dim_wh.c * sizeof(float)) );
checkCuda( cudaMallocHost(&amodel_offset, K * dim_amodel_offset.c * sizeof(float)) );
checkCuda( cudaMalloc(&track_d, K * dim_track.c * sizeof(float)) );
checkCuda( cudaMalloc(&dep_d, K * dim_dep.c * sizeof(float)) );
checkCuda( cudaMalloc(&rot_d, K * dim_rot.c * sizeof(float)) );
checkCuda( cudaMalloc(&dim_d, K * dim_dim.c * sizeof(float)) );
checkCuda( cudaMalloc(&wh_d, K * dim_wh.c * sizeof(float)) );
checkCuda( cudaMalloc(&amodel_offset_d, K * dim_amodel_offset.c * sizeof(float)) );
checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) );
for(int bi=0; bi<nBatches; bi++) {
cv::Mat calibs_ = cv::Mat::zeros(cv::Size(4,3), CV_32F);
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
calibs_.at<float>(0,0) = 633.0;
calibs_.at<float>(1,1) = 633.0;
calibs_.at<float>(2,2) = 1.0;
}
calibs_.at<float>(2,2) = 1.0;
calibs.push_back(calibs_);
}
// Alloc array used in the kernel
checkCuda( cudaMalloc(&src_out, K *sizeof(float)) );
checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) );
trRes.resize(nBatches);
countTr.resize(nBatches, 0);
trackId.resize(nBatches, 0);
return true;
}
bool CenterTrack::init_visualization(const int n_classes){
classes = n_classes;
// const char *kitti_class_name[] = {
// "person", "car", "bicycle"};
// classesNames = std::vector<std::string>(kitti_class_name, std::end( kitti_class_name));
const char *class_name[] = {"car", "truck", "bus", "trailer", "construction_vehicle", "pedestrian",
"motorcycle", "bicycle", "traffic_cone", "barrier"};
classesNames = std::vector<std::string>(class_name, std::end( class_name));
// const char *coco_class_name[] = {
// "person", "bicycle", "car", "motorcycle", "airplane",
// "bus", "train", "truck", "boat", "traffic light", "fire hydrant",
// "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse",
// "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
// "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis",
// "snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
// "skateboard", "surfboard", "tennis racket", "bottle", "wine glass",
// "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich",
// "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake",
// "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv",
// "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
// "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
// "scissors", "teddy bear", "hair drier", "toothbrush"
// };
// classesNames = std::vector<std::string>(coco_class_name, std::end( coco_class_name));
for(int c=0; c<classes; c++) {
int offset = c*123457 % classes;
float r = getColor(2, offset, classes);
float g = getColor(1, offset, classes);
float b = getColor(0, offset, classes);
colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
}
for(int c=0; c<256; c++) {
int offset = c * 123457 % 256;
float r = getColor(2, offset, 256);
float g = getColor(1, offset, 256);
float b = getColor(0, offset, 256);
trColors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
}
r = cv::Mat(cv::Size(3,3), CV_32F);
r.at<float>(0,1) = 0.0;
r.at<float>(1,0) = 0.0;
r.at<float>(1,1) = 1.0;
r.at<float>(1,2) = 0.0;
r.at<float>(2,1) = 0.0;
corners = cv::Mat(cv::Size(8,3), CV_32F);
corners.at<float>(1,0) = 0.0;
corners.at<float>(1,1) = 0.0;
corners.at<float>(1,2) = 0.0;
corners.at<float>(1,3) = 0.0;
pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F);
pts3DHomo.at<float>(3,0) = 1.0;
pts3DHomo.at<float>(3,1) = 1.0;
pts3DHomo.at<float>(3,2) = 1.0;
pts3DHomo.at<float>(3,3) = 1.0;
pts3DHomo.at<float>(3,4) = 1.0;
pts3DHomo.at<float>(3,5) = 1.0;
pts3DHomo.at<float>(3,6) = 1.0;
pts3DHomo.at<float>(3,7) = 1.0;
faceId.push_back({0,1,5,4});
faceId.push_back({1,2,6, 5});
faceId.push_back({3,0,4,7});
faceId.push_back({2,3,7,6});
// ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]);
return true;
}
void CenterTrack::_get_additional_inputs(){
//None no additional input
}
void CenterTrack::pre_inf(const int bi){
TKDNN_TSTART
tk::dnn::dataDim_t dim_aus;
pre_phase_net->infer(dim_aus, nullptr);
TKDNN_TSTOP
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
checkCuda( cudaDeviceSynchronize() );
}
void CenterTrack::preprocess(cv::Mat &frame, const int bi){
cv::Size sz = originalSize[bi];
// float scale = 1.0;
float new_height = dim.h;//sz.height * scale;
float new_width = dim.w;//sz.width * scale;
if(sz.height != szOld[bi].height && sz.width != szOld[bi].width){
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
calibs[bi].at<float>(0,2) = new_width / 2.0f;
calibs[bi].at<float>(1,2) = new_height /2.0f;
}
else {
calibs[bi].at<float>(0,0) = inputCalibs[bi].at<float>(0,0) * dim.w / sz.width;
calibs[bi].at<float>(0,2) = inputCalibs[bi].at<float>(0,2) * dim.w / sz.width;
calibs[bi].at<float>(1,1) = inputCalibs[bi].at<float>(1,1) * dim.h / sz.height;
calibs[bi].at<float>(1,2) = inputCalibs[bi].at<float>(1,2) * dim.h / sz.height;
}
float c[] = {new_width / 2.0f, new_height /2.0f};
float s[] = {static_cast<float>(dim.w), static_cast<float>(dim.h)};
// float s = new_width >= new_height ? new_width : new_height;
// ----------- get_affine_transform
// rot_rad = pi * 0 / 100 --> 0
//dim.print();
src.at<float>(0,0) = c[0];
src.at<float>(0,1) = c[1];
src.at<float>(1,0) = c[0];
src.at<float>(1,1) = c[1] + s[0] * -0.5;
dst.at<float>(0,0) = dim.w * 0.5;
dst.at<float>(0,1) = dim.h * 0.5;
dst.at<float>(1,0) = dim.w * 0.5;
dst.at<float>(1,1) = dim.h * 0.5 + dim.w * -0.5;
src.at<float>(2,0) = src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
src.at<float>(2,1) = src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
dst.at<float>(2,0) = dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
dst.at<float>(2,1) = dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
trans = cv::getAffineTransform( src, dst );
trans2 = cv::getAffineTransform( dst2, src );
trans2.convertTo(transOut, CV_32F);
}
szOld[bi] = sz;
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat im_Orig;
cv::cuda::GpuMat imageF1_d, imageF2_d;
im_Orig = cv::cuda::GpuMat(frame);
cv::cuda::resize (im_Orig, imageF1_d, cv::Size(dim.w, dim.h));
// imageF1_d = im_Orig;
checkCuda( cudaDeviceSynchronize() );
sz = imageF1_d.size();
cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR );
checkCuda( cudaDeviceSynchronize() );
imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0);
checkCuda( cudaDeviceSynchronize() );
dim2 = dim;
cv::cuda::GpuMat bgr[3];
cv::cuda::split(imageF1_d,bgr);//split source
for(int i=0; i<dim.c; i++)
checkCuda( cudaMemcpy(d_ptrs + i*dim.h * dim.w, (float*)bgr[i].data, dim.h * dim.w * sizeof(float), cudaMemcpyDeviceToDevice) );
normalize(d_ptrs, dim.c, dim.h, dim.w, mean_d, stddev_d);
checkCuda( cudaMemcpy(input_pre_inf_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
checkCuda( cudaDeviceSynchronize() );
#else
cv::Mat imageF;
resize(frame, imageF, cv::Size(dim.w, dim.h));
// imageF = frame;
sz = imageF.size();
cv::warpAffine(imageF, imageF, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR );
// cv::imshow("warp", imageF);
sz = imageF.size();
imageF.convertTo(imageF, CV_32FC3, 1/255.0);
dim2 = dim;
//split channels
cv::Mat bgr[3];
cv::split(imageF,bgr);//split source
for(int i=0; i<3; i++){
bgr[i] = bgr[i] - mean[i];
bgr[i] = bgr[i] / stddev[i];
}
for(int i=0; i<dim2.c; i++) {
int idx = i * imageF.rows * imageF.cols;
int ch = i;
memcpy((void*)&input[idx], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
}
checkCuda( cudaMemcpyAsync(input_pre_inf_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
checkCuda( cudaDeviceSynchronize() );
#endif
if(iter0) {
checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
checkCuda( cudaDeviceSynchronize() );
iter0=false;
}
pre_inf(bi);
checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
checkCuda( cudaDeviceSynchronize() );
}
cv::Mat CenterTrack::transform_preds_with_trans(float x1, float x2){
cv::Mat target_coords(cv::Size(1,3), CV_32F);
target_coords.at<float>(0,0) = x1;
target_coords.at<float>(0,1) = x2;
target_coords.at<float>(0,2) = 1.0;
return transOut * target_coords;
}
void CenterTrack::tracking(const int bi) {
std::vector<float> item_size(countDet);
std::vector<int> item_cl(countDet);
std::vector<float> dets(2*countDet);
for(int i=0; i<countDet; i++){
item_size[i] = (detRes[i].bb1.at<float>(0,0) - detRes[i].bb0.at<float>(0,0)) *
(detRes[i].bb1.at<float>(0,1) - detRes[i].bb0.at<float>(0,1));
item_cl[i] = detRes[i].cl;
dets[i*2] = detRes[i].ct.at<float>(0,0);
dets[i*2+1] = detRes[i].ct.at<float>(0,1);
}
std::vector<float> track_size(countTr[bi]);
std::vector<int> track_cl(countTr[bi]);
std::vector<float> tracks(2*countTr[bi]);
for(int i=0; i<countTr[bi]; i++){
track_size[i] = (trRes[bi][i].det_res.bb1.at<float>(0,0) - trRes[bi][i].det_res.bb0.at<float>(0,0)) *
(trRes[bi][i].det_res.bb1.at<float>(0,1) - trRes[bi][i].det_res.bb0.at<float>(0,1));
track_cl[i] = trRes[bi][i].det_res.cl;
tracks[i*2] = trRes[bi][i].det_res.ct.at<float>(0,0);
tracks[i*2+1] = trRes[bi][i].det_res.ct.at<float>(0,1);
}
std::vector<float> dist(countTr[bi]*countDet);
bool invalid;
for(int i=0; i<countTr[bi]; i++){
for(int j=0; j<countDet; j++){
dist[j*countTr[bi]+i] = pow((tracks[i*2] - dets[j*2]), 2) +
pow((tracks[i*2+1] - dets[j*2+1]), 2);
invalid = dist[j*countTr[bi]+i] > track_size[i] ||
dist[j*countTr[bi]+i] > item_size[j] ||
item_cl[j] != track_cl[i];
dist[j*countTr[bi]+i] = dist[j*countTr[bi]+i] + invalid * (1 << 18);
}
}
std::vector<int> matched_indices(2*countTr[bi]);
float min_tr;
int min_idtr = -1;
for(int i=0; i<countTr[bi]; i++) {
matched_indices[i*2] = -1;
matched_indices[i*2+1] = -1;
}
for(int i=0; i<countDet; i++){
min_tr=(1 << 18);
for(int j=0; j<countTr[bi]; j++){
if(dist[i*countTr[bi]+j]<min_tr) {
min_tr = dist[i*countTr[bi]+j];
min_idtr = j;
}
}
if(min_tr < (1<<16)) {
for(int j=0; j<countDet; j++)
dist[j*countTr[bi]+min_idtr] = (1 << 18);
matched_indices[2*min_idtr] = min_idtr;
matched_indices[2*min_idtr+1] = i;
}
}
std::vector<bool> unmatched_dets(countDet);
for(int i=0; i<countDet; i++)
unmatched_dets[i] = false;
std::vector<bool> unmatched_tracks(countTr[bi]);
for(int i=0; i<countTr[bi]; i++)
unmatched_tracks[i] = false;
for(int i=0; i<countTr[bi]; i++) {
if(matched_indices[2*i] != -1)
unmatched_tracks[matched_indices[2*i]]=true;
if(matched_indices[2*i+1] != -1)
unmatched_dets[matched_indices[2*i+1]]=true;
}
//match
for(int i=0; i<countTr[bi]; i++) {
if(matched_indices[2*i+1] != -1 && matched_indices[2*i] != -1) { //second condition is optional
int tr_id = matched_indices[2*i];
int d_id = matched_indices[2*i+1];
// trRes[tr_id].det_res = detRes[d_id];
trRes[bi][tr_id].det_res.score = detRes[d_id].score;
trRes[bi][tr_id].det_res.cl = detRes[d_id].cl;
trRes[bi][tr_id].det_res.ct = detRes[d_id].ct;
trRes[bi][tr_id].det_res.tr = detRes[d_id].tr;
trRes[bi][tr_id].det_res.bb0 = detRes[d_id].bb0;
trRes[bi][tr_id].det_res.bb1 = detRes[d_id].bb1;
trRes[bi][tr_id].det_res.dep = detRes[d_id].dep;
trRes[bi][tr_id].det_res.dim[0] = detRes[d_id].dim[0];
trRes[bi][tr_id].det_res.dim[1] = detRes[d_id].dim[1];
trRes[bi][tr_id].det_res.dim[2] = detRes[d_id].dim[2];
trRes[bi][tr_id].det_res.alpha = detRes[d_id].alpha;
trRes[bi][tr_id].det_res.x = detRes[d_id].x;
trRes[bi][tr_id].det_res.y = detRes[d_id].y;
trRes[bi][tr_id].det_res.z = detRes[d_id].z;
trRes[bi][tr_id].det_res.rot_y = detRes[d_id].rot_y;
// trRes[bi][matched_indices[2*i]].tracking_id = ; is the same
// trRes[bi][matched_indices[2*i]].color = ; is the same
trRes[bi][tr_id].age = 1;
trRes[bi][tr_id].active = trRes[bi][tr_id].active+1;
}
}
//delete target umatched track
int new_count_tr = 0;
for(int i=0; i<countTr[bi]; i++) {
if(unmatched_tracks[i])
new_count_tr++;
}
if(new_count_tr == 0 && countTr[bi] != 0) { //reset
trRes[bi].clear();
countTr[bi] = 0;
}
int old_count_tr = countTr[bi];
if(countTr[bi] != 0 && new_count_tr != countTr[bi]) {
std::vector<struct trackingRes> new_tr_res;
int id_new_tr=0;
for(int i=0; i<countTr[bi]; i++) {
if(unmatched_tracks[i]) {
struct trackingRes new_tr_res_;
// new_tr_res_new_det_res.det_res = trRes[i].det_res;
new_tr_res_.det_res.score = trRes[bi][i].det_res.score;
new_tr_res_.det_res.cl = trRes[bi][i].det_res.cl;
new_tr_res_.det_res.ct = trRes[bi][i].det_res.ct;
new_tr_res_.det_res.tr = trRes[bi][i].det_res.tr;
new_tr_res_.det_res.bb0 = trRes[bi][i].det_res.bb0;
new_tr_res_.det_res.bb1 = trRes[bi][i].det_res.bb1;
new_tr_res_.det_res.dep = trRes[bi][i].det_res.dep;
new_tr_res_.det_res.dim[0] = trRes[bi][i].det_res.dim[0];
new_tr_res_.det_res.dim[1] = trRes[bi][i].det_res.dim[1];
new_tr_res_.det_res.dim[2] = trRes[bi][i].det_res.dim[2];
new_tr_res_.det_res.alpha = trRes[bi][i].det_res.alpha;
new_tr_res_.det_res.x = trRes[bi][i].det_res.x;
new_tr_res_.det_res.y = trRes[bi][i].det_res.y;
new_tr_res_.det_res.z = trRes[bi][i].det_res.z;
new_tr_res_.det_res.rot_y = trRes[bi][i].det_res.rot_y;
new_tr_res_.tracking_id = trRes[bi][i].tracking_id;
new_tr_res_.age = trRes[bi][i].age;
new_tr_res_.active = trRes[bi][i].active;
new_tr_res_.color = trRes[bi][i].color;
id_new_tr ++;
new_tr_res.push_back(new_tr_res_);
}
}
if(countTr[bi]) {
trRes[bi].clear();
}
countTr[bi] = new_count_tr;
trRes[bi] = new_tr_res;
}
int count_tr_ = countTr[bi];
for(int i=0; i<countDet; i++) {
if((!unmatched_dets[i]) && detRes[i].score > newThresh) {
count_tr_ ++;
struct trackingRes new_tr_res_;
new_tr_res_.det_res.score = detRes[i].score;
new_tr_res_.det_res.cl = detRes[i].cl;
new_tr_res_.det_res.ct = detRes[i].ct;
new_tr_res_.det_res.tr = detRes[i].tr;
new_tr_res_.det_res.bb0 = detRes[i].bb0;
new_tr_res_.det_res.bb1 = detRes[i].bb1;
new_tr_res_.det_res.dep = detRes[i].dep;
new_tr_res_.det_res.dim[0] = detRes[i].dim[0];
new_tr_res_.det_res.dim[1] = detRes[i].dim[1];
new_tr_res_.det_res.dim[2] = detRes[i].dim[2];
new_tr_res_.det_res.alpha = detRes[i].alpha;
new_tr_res_.det_res.x = detRes[i].x;
new_tr_res_.det_res.y = detRes[i].y;
new_tr_res_.det_res.z = detRes[i].z;
new_tr_res_.det_res.rot_y = detRes[i].rot_y;
new_tr_res_.tracking_id = trackId[bi]++;
new_tr_res_.age = 1;
new_tr_res_.active = 1;
new_tr_res_.color = rand() % 256;
if(trRes.size() <= bi) {
std::vector<struct trackingRes> v_new_tr_res_;
v_new_tr_res_.push_back(new_tr_res_);
trRes.push_back(v_new_tr_res_);
}
else
trRes[bi].push_back(new_tr_res_);
}
}
countTr[bi] = count_tr_;
//reset the tracker id
if(trackId[bi] == 1000)
trackId[bi] = 0;
detRes.clear();
}
void CenterTrack::postprocess(const int bi, const bool mAP) {
dnnType *rt_out[9];
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi;
rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi;
rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi;
rt_out[7] = (dnnType *)netRT->buffersRT[8]+ netRT->buffersDIM[8].tot()*bi;
rt_out[8] = (dnnType *)netRT->buffersRT[9]+ netRT->buffersDIM[9].tot()*bi;
// ------------------------------------ process --------------------------------------------
activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot());
checkCuda( cudaDeviceSynchronize() );
// output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1.
activationSIGMOIDForward(rt_out[5], rt_out[5], dim_dep.tot());
checkCuda( cudaDeviceSynchronize() );
transformDep(ones, ones + dim_dep.tot(), rt_out[5], rt_out[5] + dim_dep.tot());
checkCuda( cudaDeviceSynchronize() );
// nms
subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op);
// ----------- nms end
// ----------- topk
if(K > dim_hm.h * dim_hm.w){
printf ("Error topk (K is too large)\n");
return;
}
checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );
sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d);
checkCuda( cudaDeviceSynchronize() );
topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d);
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) );
topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d);
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(intxs, inttopk_xs_d, K * sizeof(int), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(intys, inttopk_ys_d, K * sizeof(int), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
// ----------- topk end
topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out);
checkCuda( cudaDeviceSynchronize() );
bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out);
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(bby0, bby0_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(bbx1, bbx1_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
//regression heads
// ['tracking', 'dep', 'rot', 'dim', 'amodel_offset',
// 'nuscenes_att', 'velocity']
getRecordsFromTopKId(topk_inds_d, K, dim_track.c, dim_track.h * dim_track.w, rt_out[4], track_d, ids_out);
checkCuda( cudaMemcpy(track, track_d, K * dim_track.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[5], dep_d, ids_out);
checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[6], rot_d, ids_out);
checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[7], dim_d, ids_out);
checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_amodel_offset.c, dim_amodel_offset.h * dim_amodel_offset.w, rt_out[8], amodel_offset_d, ids_out);
checkCuda( cudaMemcpy(amodel_offset, amodel_offset_d, K * dim_amodel_offset.c * sizeof(float), cudaMemcpyDeviceToHost) );
// ---------------------------------- post-process -----------------------------------------
countDet = 0;
detRes.clear();
for(int i=0; i<K; i++){
if(scores[i] < outThresh)
break;
countDet ++;
struct detectionRes new_det_res;
new_det_res.score = scores[i];
new_det_res.cl = clses[i]+1;
// ret_s=scores[i];
// ret_c=clses[i]+1;
new_det_res.ct = transform_preds_with_trans(intxs[i], intys[i]);
new_det_res.tr = transform_preds_with_trans(intxs[i] + track[i], intys[i] + track[i+K]);
new_det_res.tr = new_det_res.tr -new_det_res.ct;
new_det_res.bb0 = transform_preds_with_trans(bbx0[i], bby0[i]);
new_det_res.bb1 = transform_preds_with_trans(bbx1[i], bby1[i]);
new_det_res.ct = transform_preds_with_trans(((bbx0[i]+bbx1[i])/2 + amodel_offset[i]),
((bby0[i]+bby1[i])/2 + amodel_offset[i+K]));
new_det_res.dep = dep[i];
new_det_res.dim[0] = dim_[i];
new_det_res.dim[1] = dim_[i+K];
new_det_res.dim[2] = dim_[i+2*K];
// unproject_2d_to_3d
new_det_res.z = dep[i] - calibs[bi].at<float>(2,3);
new_det_res.x = ((float)new_det_res.ct.at<float>(0,0) * dep[i] - calibs[bi].at<float>(0,3) -
calibs[bi].at<float>(0,2) * new_det_res.z) / calibs[bi].at<float>(0,0);
new_det_res.y = ((float)new_det_res.ct.at<float>(0,1) * dep[i] - calibs[bi].at<float>(1,3) -
calibs[bi].at<float>(1,2) * new_det_res.z) / calibs[bi].at<float>(1,1) + (dim_[i] / 2);
// alpha2rot_y
// idx = rot[:, 1] > rot[:, 5]
// alpha1 = np.arctan2(rot[:, 2], rot[:, 3]) + (-0.5 * np.pi)
// alpha2 = np.arctan2(rot[:, 6], rot[:, 7]) + ( 0.5 * np.pi)
// return alpha1 * idx + alpha2 * (1 - idx)
if(rot[1*K + i] > rot[5*K + i])
new_det_res.alpha = std::atan2(rot[2*K + i], rot[3*K + i]) -0.5 * M_PI;
else
new_det_res.alpha = std::atan2(rot[6*K + i], rot[7*K + i]) +0.5 * M_PI;
new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at<float>(0,0) - calibs[bi].at<float>(0,2), calibs[bi].at<float>(0,0)));
new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest
detRes.push_back(new_det_res);
}
// track step
tracking(bi);
}
void CenterTrack::draw(std::vector<cv::Mat>& frames) {
struct trackingRes t;
float sc;
int id;
std::string txt;
int baseline = 0;
float font_scale = 0.8;
int thickness = 2;
for(int bi=0; bi<frames.size(); ++bi) {
float scale_x = float(originalSize[bi].width)/dim.w;
float scale_y = float(originalSize[bi].height)/dim.h;
resize(frames[bi], frames[bi], originalSize[bi]);
// draw dets
for(int i=0; trRes.size() != 0 && i<trRes[bi].size(); i++) {
t = trRes[bi][i];
id = t.tracking_id;
txt = classesNames[t.det_res.cl-1]+'-'+std::to_string(id); //forse ha bisogno di cl-1
cv::Size text_size = getTextSize(txt, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
if(t.det_res.score > confThreshold){// && t.active!=0) {
if(!mode3D) {
cv::rectangle(frames[bi],
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x, t.det_res.bb0.at<float>(0,1) * scale_y),
cv::Point(t.det_res.bb1.at<float>(0,0) * scale_x, t.det_res.bb1.at<float>(0,1) * scale_y),
trColors[t.color], thickness);
cv::rectangle(frames[bi],
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x, t.det_res.bb0.at<float>(0,1) * scale_y - text_size.height - thickness),
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x + text_size.width, t.det_res.bb0.at<float>(0,1) * scale_y),
trColors[t.color], -1);
cv::putText(frames[bi], txt,
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x, t.det_res.bb0.at<float>(0,1) * scale_y - thickness -1),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1);
cv::arrowedLine(frames[bi],
cv::Point((int)t.det_res.ct.at<float>(0,0) * scale_x, (int)t.det_res.ct.at<float>(0,1) * scale_y),
cv::Point((int)(t.det_res.ct.at<float>(0,0) * scale_x + t.det_res.tr.at<float>(0,0) * scale_x),
(int)(t.det_res.ct.at<float>(0,1) * scale_y + t.det_res.tr.at<float>(0,1) * scale_y)),
cv::Scalar(255, 0, 255), 2);
}
//3d
if(mode3D && t.det_res.z > 1){
r.at<float>(0,0) = std::cos(t.det_res.rot_y);
r.at<float>(0,2) = std::sin(t.det_res.rot_y);
r.at<float>(2,0) = -std::sin(t.det_res.rot_y);
r.at<float>(2,2) = std::cos(t.det_res.rot_y);
corners.at<float>(0,0) = t.det_res.dim[2]/2;
corners.at<float>(0,1) = t.det_res.dim[2]/2;
corners.at<float>(0,2) = -t.det_res.dim[2]/2;
corners.at<float>(0,3) = -t.det_res.dim[2]/2;
corners.at<float>(0,4) = t.det_res.dim[2]/2;
corners.at<float>(0,5) = t.det_res.dim[2]/2;
corners.at<float>(0,6) = -t.det_res.dim[2]/2;
corners.at<float>(0,7) = -t.det_res.dim[2]/2;
corners.at<float>(1,4) = -t.det_res.dim[0];
corners.at<float>(1,5) = -t.det_res.dim[0];
corners.at<float>(1,6) = -t.det_res.dim[0];
corners.at<float>(1,7) = -t.det_res.dim[0];
corners.at<float>(2,0) = t.det_res.dim[1]/2;
corners.at<float>(2,1) = -t.det_res.dim[1]/2;
corners.at<float>(2,2) = -t.det_res.dim[1]/2;
corners.at<float>(2,3) = t.det_res.dim[1]/2;
corners.at<float>(2,4) = t.det_res.dim[1]/2;
corners.at<float>(2,5) = -t.det_res.dim[1]/2;
corners.at<float>(2,6) = -t.det_res.dim[1]/2;
corners.at<float>(2,7) = t.det_res.dim[1]/2;
cv::Mat aus = r * corners;
for(int k=0; k<8; k++) {
aus.at<float>(0,k) += t.det_res.x;
aus.at<float>(1,k) += t.det_res.y;
aus.at<float>(2,k) += t.det_res.z;
}
// corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3)));
for(int k1=0; k1<3; k1++) {
for(int k2=0; k2<8; k2++)
pts3DHomo.at<float>(k1,k2) = aus.at<float>(k1,k2);
}
aus.release();
aus = calibs[bi] * pts3DHomo;
std::vector<float> res_corners;
for(int k=0; k<8; k++) {
res_corners.push_back(aus.at<float>(0,k) / aus.at<float>(2,k));
res_corners.push_back(aus.at<float>(1,k) / aus.at<float>(2,k));
}
aus.release();
for(int ind_f=3; ind_f>=0; ind_f--) {
for(int j=0; j<4; j++) {
cv::line(frames[bi],
cv::Point((int)res_corners.at(faceId.at(ind_f).at(j) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(j) * 2 + 1) * scale_y),
cv::Point((int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1) * scale_y),
trColors[t.color], 2);
if(ind_f == 0 && j==3) {
cv::line(frames[bi],
cv::Point((int)res_corners.at(faceId.at(ind_f).at(0) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(0) * 2 + 1) * scale_y),
cv::Point((int)res_corners.at(faceId.at(ind_f).at(2) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(2) * 2 + 1) * scale_y), trColors[t.color], 2);
cv::line(frames[bi],
cv::Point((int)res_corners.at(faceId.at(ind_f).at(1) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(1) * 2 + 1) * scale_y),
cv::Point((int)res_corners.at(faceId.at(ind_f).at(3) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(3) * 2 + 1) * scale_y), trColors[t.color], 2);
}
}
}
float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0;
for(int k=0; k<8; k++) {
if(res_corners[2*k] < bb0)
bb0 = res_corners[2*k];
if(res_corners[2*k] > bb1)
bb1 = res_corners[2*k];
if(res_corners[2*k+1] < bb2)
bb2 = res_corners[2*k+1];
if(res_corners[2*k+1] > bb3)
bb3 = res_corners[2*k+1];
}
// if(not no_bbox):
// cv::rectangle(frame,
// cv::Point(bb0, bb2),
// cv::Point(bb1, bb3),
// trColors[t.color], thickness);
cv::rectangle(frames[bi],
cv::Point(bb0 * scale_x, bb2 * scale_y - text_size.height - thickness),
cv::Point(bb0 * scale_x + text_size.width, bb2 * scale_y),
trColors[t.color], -1);
cv::putText(frames[bi], txt,
cv::Point(bb0 * scale_x, bb2 * scale_y - thickness -1),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1);
cv::arrowedLine(frames[bi],
cv::Point((int)((bb0 + bb1)/2) * scale_x, (int)((bb2 + bb3)/2) * scale_y),
cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at<float>(0,0)) * scale_x,
(int)((bb2 + bb3)/2 + t.det_res.tr.at<float>(0,1)) * scale_y),
cv::Scalar(255, 0, 255), 2);
}
}
}
}
}
}}
src/CenternetDetection.cpp
+9 -12
View File
@@ -3,12 +3,11 @@
namespace tk { namespace dnn {
bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){
bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
classes = n_classes;
nBatches = n_batches;
confThreshold = conf_thresh;
dim = netRT->input_dim;
@@ -118,9 +117,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe
dst2.at<float>(2,0)=dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
dst2.at<float>(2,1)=dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
return true;
return true;
}
@@ -349,21 +346,21 @@ void CenternetDetection::postprocess(const int bi, const bool mAP){
new_pt1.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*bbx0[i] +
static_cast<float>(trans2.at<double>(0,1))*bby0[i] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt1.at<float>(1,0)=static_cast<float>(trans2.at<double>(1,0))*bbx0[i] +
new_pt1.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*bbx0[i] +
static_cast<float>(trans2.at<double>(1,1))*bby0[i] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
new_pt2.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*bbx1[i] +
static_cast<float>(trans2.at<double>(0,1))*bby1[i] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt2.at<float>(1,0)=static_cast<float>(trans2.at<double>(1,0))*bbx1[i] +
new_pt2.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*bbx1[i] +
static_cast<float>(trans2.at<double>(1,1))*bby1[i] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
target_coords[i*4] = new_pt1.at<float>(0,0);
target_coords[i*4+1] = new_pt1.at<float>(1,0);
target_coords[i*4+1] = new_pt1.at<float>(0,1);
target_coords[i*4+2] = new_pt2.at<float>(0,0);
target_coords[i*4+3] = new_pt2.at<float>(1,0);
target_coords[i*4+3] = new_pt2.at<float>(0,1);
}
detected.clear();
@@ -374,10 +371,10 @@ void CenternetDetection::postprocess(const int bi, const bool mAP){
// std::cout<<"th: "<<scores[j]<<" - cl: "<<clses[j]<<" i: "<<i<<std::endl;
//add coco bbox
//det[0:4], i, det[4]
float x0 = target_coords[j*4];
float y0 = target_coords[j*4+1];
float x1 = target_coords[j*4+2];
float y1 = target_coords[j*4+3];
int x0 = target_coords[j*4];
int y0 = target_coords[j*4+1];
int x1 = target_coords[j*4+2];
int y1 = target_coords[j*4+3];
int obj_class = clses[j];
float prob = scores[j];
// std::cout<<"("<<x0<<", "<<y0<<"),("<<x1<<", "<<y1<<")"<<std::endl;
src/CenternetDetection3D.cpp
-540
View File
@@ -1,540 +0,0 @@
#include "CenternetDetection3D.h"
namespace tk { namespace dnn {
bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches,
const float conf_thresh, const std::vector<cv::Mat>& k_calibs) {
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
classes = n_classes;
nBatches = n_batches;
confThreshold = conf_thresh;
inputCalibs = k_calibs;
dim = netRT->input_dim;
const char *kitti_class_name[] = {
"person", "car", "bicycle"};
classesNames = std::vector<std::string>(kitti_class_name, std::end( kitti_class_name));
for(int c=0; c<classes; c++) {
int offset = c*123457 % classes;
float r = getColor(2, offset, classes);
float g = getColor(1, offset, classes);
float b = getColor(0, offset, classes);
colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
}
src = cv::Mat(cv::Size(2,3), CV_32F);
dst = cv::Mat(cv::Size(2,3), CV_32F);
dst2 = cv::Mat(cv::Size(2,3), CV_32F);
trans = cv::Mat(cv::Size(3,2), CV_32F);
trans2 = cv::Mat(cv::Size(3,2), CV_32F);
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
dim_hm = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1);
dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1);
dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
for(int i =0; i<dim_hm.c * dim_hm.h * dim_hm.w; i++){
ids_[i] = i;
}
checkCuda( cudaMalloc(&ones, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
float *ones_h;
checkCuda( cudaMallocHost(&ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
for(int i=0; i<dim_dep.c * dim_dep.h * dim_dep.w; i++)
ones_h[i]=1.0f;
checkCuda( cudaMemcpy(ones, ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float), cudaMemcpyHostToDevice) );
checkCuda( cudaFreeHost(ones_h) );
checkCuda( cudaMallocHost(&scores, K *sizeof(float)) );
checkCuda( cudaMalloc(&scores_d, K *sizeof(float)) );
checkCuda( cudaMallocHost(&clses, K *sizeof(int)) );
checkCuda( cudaMalloc(&clses_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_inds_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_d, K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_d, K *sizeof(float)) );
checkCuda( cudaMalloc(&inttopk_ys_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&inttopk_xs_d, K *sizeof(int)) );
checkCuda( cudaMallocHost(&xs, K * sizeof(float)) );
checkCuda( cudaMallocHost(&ys, K * sizeof(float)) );
checkCuda( cudaMallocHost(&dep, K * dim_dep.c * sizeof(float)) );
checkCuda( cudaMallocHost(&rot, K * dim_rot.c * sizeof(float)) );
checkCuda( cudaMallocHost(&dim_, K * dim_dim.c * sizeof(float)) );
checkCuda( cudaMallocHost(&wh, K * dim_wh.c * sizeof(float)) );
checkCuda( cudaMalloc(&dep_d, K * dim_dep.c * sizeof(float)) );
checkCuda( cudaMalloc(&rot_d, K * dim_rot.c * sizeof(float)) );
checkCuda( cudaMalloc(&dim_d, K * dim_dim.c * sizeof(float)) );
checkCuda( cudaMalloc(&wh_d, K * dim_wh.c * sizeof(float)) );
checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) );
#ifdef OPENCV_CUDACONTRIB
checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) );
checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) );
float mean[3] = {0.485, 0.456, 0.406};
float stddev[3] = {0.229, 0.224, 0.225};
checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
#else
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
mean << 0.485, 0.456, 0.406;
stddev << 0.229, 0.224, 0.225;
#endif
for(int bi=0; bi<nBatches; bi++) {
cv::Mat calibs_ = cv::Mat::zeros(cv::Size(4,3), CV_32F);
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
calibs_.at<float>(0,0) = 707.0493;
calibs_.at<float>(0,2) = 604.0814;
calibs_.at<float>(1,1) = 707.0493;
calibs_.at<float>(1,2) = 180.5066;
calibs_.at<float>(0,3) = 45.75831;
calibs_.at<float>(1,3) = -0.3454157;
calibs_.at<float>(2,2) = 1.0;
calibs_.at<float>(2,3) = 0.004981016;
}
else {
calibs_.at<float>(0,0) = inputCalibs[bi].at<float>(0,0);// * (1440.0/dim.w);// / 1440;
calibs_.at<float>(0,2) = inputCalibs[bi].at<float>(0,2);// * (1440.0/dim.w);// / 1440;
calibs_.at<float>(1,1) = inputCalibs[bi].at<float>(1,1);// * (1080.0/dim.h);//dim.h / 1080;
calibs_.at<float>(1,2) = inputCalibs[bi].at<float>(1,2);// * (1080.0/dim.h);//dim.h / 1080;
calibs_.at<float>(2,2) = 1.0;
}
// calibs_.at<float>(0,3) = 45.75831;
// calibs_.at<float>(1,3) = -0.3454157;
// calibs_.at<float>(2,2) = 1.0;
// calibs_.at<float>(2,3) = 0.004981016;
calibs.push_back(calibs_);
}
r = cv::Mat(cv::Size(3,3), CV_32F);
r.at<float>(0,1) = 0.0;
r.at<float>(1,0) = 0.0;
r.at<float>(1,1) = 1.0;
r.at<float>(1,2) = 0.0;
r.at<float>(2,1) = 0.0;
corners = cv::Mat(cv::Size(8,3), CV_32F);
corners.at<float>(1,0) = 0.0;
corners.at<float>(1,1) = 0.0;
corners.at<float>(1,2) = 0.0;
corners.at<float>(1,3) = 0.0;
pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F);
pts3DHomo.at<float>(3,0) = 1.0;
pts3DHomo.at<float>(3,1) = 1.0;
pts3DHomo.at<float>(3,2) = 1.0;
pts3DHomo.at<float>(3,3) = 1.0;
pts3DHomo.at<float>(3,4) = 1.0;
pts3DHomo.at<float>(3,5) = 1.0;
pts3DHomo.at<float>(3,6) = 1.0;
pts3DHomo.at<float>(3,7) = 1.0;
checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) );
// Alloc array used in the kernel
checkCuda( cudaMalloc(&srcOut, K *sizeof(float)) );
checkCuda( cudaMalloc(&idsOut, K *sizeof(int)) );
dst2.at<float>(0,0)=width * 0.5;
dst2.at<float>(0,1)=width * 0.5;
dst2.at<float>(1,0)=width * 0.5;
dst2.at<float>(1,1)=width * 0.5 + width * -0.5;
dst2.at<float>(2,0)=dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
dst2.at<float>(2,1)=dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
faceId.push_back({0,1,5,4});
faceId.push_back({1,2,6, 5});
faceId.push_back({2,3,7,6});
faceId.push_back({3,0,4,7});
// ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]);
return true;
}
void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){
cv::Size sz = originalSize[bi];
float new_height = dim.h;//sz.height * scale;
float new_width = dim.w;//sz.width * scale;
if(sz.height != sz_old.height && sz.width != sz_old.width){
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
calibs[bi].at<float>(0,2) = new_width / 2.0f;
calibs[bi].at<float>(1,2) = new_height /2.0f;
}
else {
calibs[bi].at<float>(0,0) = inputCalibs[bi].at<float>(0,0) * 2.0 * dim.w / sz.width;
calibs[bi].at<float>(0,2) = inputCalibs[bi].at<float>(0,2) * dim.w / sz.width ;
calibs[bi].at<float>(1,1) = inputCalibs[bi].at<float>(1,1) * 2.0 * dim.h / sz.height;
calibs[bi].at<float>(1,2) = inputCalibs[bi].at<float>(1,2) * dim.h / sz.height;
}
float c[] = {new_width / 2.0f, new_height /2.0f};
float s[] = {new_width, new_height};
// ----------- get_affine_transform
// rot_rad = pi * 0 / 100 --> 0
src.at<float>(0,0)=c[0];
src.at<float>(0,1)=c[1];
src.at<float>(1,0)=c[0];
src.at<float>(1,1)=c[1] + s[0] * -0.5;
dst.at<float>(0,0)=netRT->input_dim.w * 0.5;
dst.at<float>(0,1)=netRT->input_dim.h * 0.5;
dst.at<float>(1,0)=netRT->input_dim.w * 0.5;
dst.at<float>(1,1)=netRT->input_dim.h * 0.5 + netRT->input_dim.w * -0.5;
src.at<float>(2,0)=src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
src.at<float>(2,1)=src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
dst.at<float>(2,0)=dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
dst.at<float>(2,1)=dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
trans = cv::getAffineTransform( src, dst );
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME gett affine trans: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
trans2 = cv::getAffineTransform( dst2, src );
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
}
sz_old = sz;
#ifdef OPENCV_CUDACONTRIB
// std::cout<<"OPENCV CPMTROB\n";
cv::cuda::GpuMat im_Orig;
cv::cuda::GpuMat imageF1_d, imageF2_d;
im_Orig = cv::cuda::GpuMat(frame);
cv::cuda::resize (im_Orig, imageF1_d, cv::Size(dim.w, dim.h));//cv::Size(new_width, new_height));
// imageF1_d = im_Orig;
checkCuda( cudaDeviceSynchronize() );
sz = imageF1_d.size();
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME resize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR );
checkCuda( cudaDeviceSynchronize() );
imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0);
checkCuda( cudaDeviceSynchronize() );
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME convert: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
dim2 = dim;
cv::cuda::GpuMat bgr[3];
cv::cuda::split(imageF1_d,bgr);//split source
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME split: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
for(int i=0; i<dim.c; i++)
checkCuda( cudaMemcpy(d_ptrs + i*dim.h * dim.w, (float*)bgr[i].data, dim.h * dim.w * sizeof(float), cudaMemcpyDeviceToDevice) );
normalize(d_ptrs, dim.c, dim.h, dim.w, mean_d, stddev_d);
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME normalize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
#else
// std::cout<<"NO OPENCV CPMTROB\n";
cv::Mat imageF;
resize(frame, imageF, cv::Size(dim.w, dim.h));//cv::Size(new_width, new_height));
// imageF = frame;
sz = imageF.size();
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME resize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
cv::Mat trans = cv::getAffineTransform( src, dst );
cv::warpAffine(imageF, imageF, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR );
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME warpAffine: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
sz = imageF.size();
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
imageF.convertTo(imageF, CV_32FC3, 1/255.0);
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME convertto: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
dim2 = dim;
//split channels
cv::Mat bgr[3];
cv::split(imageF,bgr);//split source
for(int i=0; i<3; i++){
bgr[i] = bgr[i] - mean[i];
bgr[i] = bgr[i] / stddev[i];
}
//write channels
for(int i=0; i<dim2.c; i++) {
int idx = i*imageF.rows*imageF.cols;
int ch = dim2.c-3 +i;
// std::cout<<"i: "<<i<<", idx: "<<idx<<", ch: "<<ch<<std::endl;
memcpy((void*)&input[idx+ netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
#endif
}
void CenternetDetection3D::postprocess(const int bi, const bool mAP) {
dnnType *rt_out[7];
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi;
rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi;
rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi;
// ------------------------------------ process --------------------------------------------
activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot());
checkCuda( cudaDeviceSynchronize() );
// output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1.
activationSIGMOIDForward(rt_out[4], rt_out[4], dim_dep.tot());
checkCuda( cudaDeviceSynchronize() );
transformDep(ones, ones + dim_dep.tot(), rt_out[4], rt_out[4] + dim_dep.tot());
checkCuda( cudaDeviceSynchronize() );
subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op);
// ----------- nms end
// ----------- topk
if(K > dim_hm.h * dim_hm.w){
printf ("Error topk (K is too large)\n");
return;
}
checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );
sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d);
checkCuda( cudaDeviceSynchronize() );
topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d);
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) );
topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d);
checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
// ----------- topk end
topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], srcOut, idsOut);
// checkCuda( cudaDeviceSynchronize() );
getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[4], dep_d, idsOut);
checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[5], rot_d, idsOut);
checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[6], dim_d, idsOut);
checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_wh.c, dim_wh.h * dim_wh.w, rt_out[2], wh_d, idsOut);
checkCuda( cudaMemcpy(wh, wh_d, K * dim_wh.c * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(xs, topk_xs_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(ys, topk_ys_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
// ---------------------------------- post-process -----------------------------------------
// ddd_post_process_2d
cv::Mat new_pt1(cv::Size(1,2), CV_32F);
cv::Mat new_pt2(cv::Size(1,2), CV_32F);
for(int i = 0; i<K; i++){
new_pt1.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*xs[i] +
static_cast<float>(trans2.at<double>(0,1))*ys[i] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt1.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*xs[i] +
static_cast<float>(trans2.at<double>(1,1))*ys[i] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
new_pt2.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*wh[i] +
static_cast<float>(trans2.at<double>(0,1))*wh[K+i] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt2.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*wh[i] +
static_cast<float>(trans2.at<double>(1,1))*wh[K+i] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
target_coords[i*4] = new_pt1.at<float>(0,0);
target_coords[i*4+1] = new_pt1.at<float>(0,1);
target_coords[i*4+2] = new_pt2.at<float>(0,0);
target_coords[i*4+3] = new_pt2.at<float>(0,1);
}
float alpha;
float x, y, z, rot_y;
detected3D.clear();
for(int i = 0; i<classes; i++){
for(int j=0; j<K; j++){
if(clses[j] == i){
//get alpha
if(rot[1*K + j] > rot[5*K + j])
alpha = std::atan2(rot[2*K + j], rot[3*K + j]) -0.5 * M_PI;
else
alpha = std::atan2(rot[6*K + j], rot[7*K + j]) +0.5 * M_PI;
// unproject_2d_to_3d
z = dep[j] - calibs[bi].at<float>(2,3);// z = depth - P[2, 3]
x = (target_coords[j*4] * dep[j] - calibs[bi].at<float>(0,3) - calibs[bi].at<float>(0,2) * z) / calibs[bi].at<float>(0,0);
y = (target_coords[j*4+1] * dep[j] - calibs[bi].at<float>(1,3) - calibs[bi].at<float>(1,2) * z) / calibs[bi].at<float>(1,1) + (dim_[j] / 2);
// alpha2rot_y
rot_y = (alpha + std::atan2(target_coords[j*4] - calibs[bi].at<float>(0,2), calibs[bi].at<float>(0,0)));
if(rot_y>M_PI)
rot_y -= 2*M_PI;
if(rot_y<M_PI)
rot_y += 2*M_PI;
if(scores[j] > confThreshold) {
if(z>0) {
// compute_box_3d
r.at<float>(0,0) = std::cos(rot_y);
r.at<float>(0,2) = std::sin(rot_y);
r.at<float>(2,0) = -std::sin(rot_y);
r.at<float>(2,2) = std::cos(rot_y);
corners.at<float>(0,0) = dim_[2*K+j]/2;
corners.at<float>(0,1) = dim_[2*K+j]/2;
corners.at<float>(0,2) = -dim_[2*K+j]/2;
corners.at<float>(0,3) = -dim_[2*K+j]/2;
corners.at<float>(0,4) = dim_[2*K+j]/2;
corners.at<float>(0,5) = dim_[2*K+j]/2;
corners.at<float>(0,6) = -dim_[2*K+j]/2;
corners.at<float>(0,7) = -dim_[2*K+j]/2;
corners.at<float>(1,4) = -dim_[j];
corners.at<float>(1,5) = -dim_[j];
corners.at<float>(1,6) = -dim_[j];
corners.at<float>(1,7) = -dim_[j];
corners.at<float>(2,0) = dim_[K+j]/2;
corners.at<float>(2,1) = -dim_[K+j]/2;
corners.at<float>(2,2) = -dim_[K+j]/2;
corners.at<float>(2,3) = dim_[K+j]/2;
corners.at<float>(2,4) = dim_[K+j]/2;
corners.at<float>(2,5) = -dim_[K+j]/2;
corners.at<float>(2,6) = -dim_[K+j]/2;
corners.at<float>(2,7) = dim_[K+j]/2;
cv::Mat aus = r * corners;
for(int k=0; k<8; k++) {
aus.at<float>(0,k) += x;
aus.at<float>(1,k) += y;
aus.at<float>(2,k) += z;
}
// corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3)));
for(int k1=0; k1<3; k1++) {
for(int k2=0; k2<8; k2++)
pts3DHomo.at<float>(k1,k2) = aus.at<float>(k1,k2);
}
aus.release();
aus = calibs[bi] * pts3DHomo;
tk::dnn::box3D res;
for(int k=0; k<8; k++) {
res.corners.push_back(aus.at<float>(0,k) / aus.at<float>(2,k));
res.corners.push_back(aus.at<float>(1,k) / aus.at<float>(2,k));
}
res.cl = i;
res.prob = scores[j];
//res.print();
detected3D.push_back(res);
}
}
}
}
}
batchDetected.push_back(detected3D);
}
void CenternetDetection3D::draw(std::vector<cv::Mat>& frames) {
tk::dnn::box3D b;
int x0, w, x1, y0, h, y1;
int objClass;
std::string det_class;
int baseline = 0;
float font_scale = 0.5;
int thickness = 2;
for(int bi=0; bi<frames.size(); ++bi){
float scale_x = float(originalSize[bi].width)/dim.w;
float scale_y = float(originalSize[bi].height)/dim.h;
resize(frames[bi], frames[bi], originalSize[bi]);
// draw dets
for(int i=0; i<batchDetected[bi].size(); i++) {
b = batchDetected[bi][i];
for(int ind_f = 3; ind_f>=0; ind_f--) {
for(int j=0; j<4; j++) {
cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(j) * 2) * scale_x,
b.corners.at(faceId.at(ind_f).at(j) * 2 + 1) * scale_y),
cv::Point(b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2) * scale_x,
b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1) * scale_y),
colors[b.cl], 2);
if(ind_f == 0) {
cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(0) * 2) * scale_x,
b.corners.at(faceId.at(ind_f).at(0) * 2 + 1)* scale_y),
cv::Point(b.corners.at(faceId.at(ind_f).at(2) * 2) * scale_x,
b.corners.at(faceId.at(ind_f).at(2) * 2 + 1) * scale_y), colors[b.cl], 2);
cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(1) * 2)* scale_x,
b.corners.at(faceId.at(ind_f).at(1) * 2 + 1)* scale_y),
cv::Point(b.corners.at(faceId.at(ind_f).at(3) * 2)* scale_x,
b.corners.at(faceId.at(ind_f).at(3) * 2 + 1)* scale_y), colors[b.cl], 2);
}
}
}
// draw label
cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
cv::rectangle(frames[bi], cv::Point(b.corners.at(faceId.at(0).at(0) * 2)* scale_x,
b.corners.at(faceId.at(0).at(0) * 2 + 1)* scale_y),
cv::Point((b.corners.at(faceId.at(0).at(0) * 2)* scale_x + text_size.width - 2),
(b.corners.at(faceId.at(0).at(0) * 2 + 1)* scale_y - text_size.height - 2)), colors[b.cl], -1);
cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(faceId.at(0).at(0) * 2)* scale_x,
(b.corners.at(faceId.at(0).at(0) * 2 + 1)* scale_y - (baseline / 2))),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness);
}
}
}
}}
src/Conv2d.cpp
-5
View File
@@ -166,11 +166,6 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
}
initCUDNN(deConv);
if(this->groups != 1)
MACC = kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
else
MACC = input_dim.c*kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
// allocate warkspace
if (ws_sizeInBytes!=0) {
checkCuda( cudaMalloc(&workSpace, ws_sizeInBytes) );
src/DarknetParser.cpp
+7 -154
View File
@@ -17,8 +17,8 @@ namespace tk { namespace dnn {
if(sep == std::string::npos)
return false;
name = line.substr(0, sep);
value = line.substr(sep+1, line.size() - (sep+1));
name = line.substr(0, sep);
value = line.substr(sep+1, line.size() - (sep+1));
return true;
}
@@ -32,25 +32,12 @@ namespace tk { namespace dnn {
return values;
}
std::vector<float> fromStringToFloatVec(const std::string& line, const char delimiter){
std::stringstream linestream(line);
std::string value;
std::vector<float> values;
while(getline(linestream,value,delimiter))
values.push_back(std::stof(value));
return values;
}
bool darknetParseFields(const std::string& line, darknetFields_t& fields){
std::string name,value;
if(!divideNameAndValue(line, name, value))
return false;
if(name.find("new_coords") != std::string::npos)
fields.new_coords = std::stoi(value);
else if(name.find("width") != std::string::npos)
if(name.find("width") != std::string::npos)
fields.width = std::stoi(value);
else if(name.find("height") != std::string::npos)
fields.height = std::stoi(value);
@@ -92,13 +79,6 @@ namespace tk { namespace dnn {
fields.group_id = std::stoi(value);
else if(name.find("scale_x_y") != std::string::npos)
fields.scale_xy = std::stof(value);
else if(name.find("beta_nms") != std::string::npos)
fields.nms_thresh = std::stof(value);
else if(name.find("nms_kind") != std::string::npos){
if(value == "greedynms") fields.nms_kind = 0;
else if(value == "diounms") fields.nms_kind = 1;
else std::cout<<"Not supported nms_kind "<<value<<", setting to greedynms"<<std::endl;
}
else if(name.find("from") != std::string::npos)
fields.layers.push_back(std::stof(value));
else if(name.find("mask") != std::string::npos){
@@ -181,7 +161,7 @@ namespace tk { namespace dnn {
} else if(f.type == "yolo") {
std::string wgs = wgs_path + "/g" + std::to_string(netLayers.size()) + ".bin";
//printf("%d %d %s %d %f\n", f.classes, f.num/f.n_mask, wgs.c_str(), f.n_mask, f.scale_xy);
tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy, f.nms_thresh, (tk::dnn::Yolo::nmsKind_t) f.nms_kind, f.new_coords);
tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy);
if(names.size() != f.classes)
FatalError("Mismatch between number of classes and names");
l->classesNames = names;
@@ -197,7 +177,6 @@ namespace tk { namespace dnn {
if(f.activation == "relu") act = tkdnnActivationMode_t(CUDNN_ACTIVATION_RELU);
else if(f.activation == "leaky") act = tk::dnn::ACTIVATION_LEAKY;
else if(f.activation == "mish") act = tk::dnn::ACTIVATION_MISH;
else if(f.activation == "logistic") act = tk::dnn::ACTIVATION_LOGISTIC;
else { FatalError("activation not supported: " + f.activation); }
netLayers[netLayers.size()-1] = new tk::dnn::Activation(net, act);
};
@@ -222,7 +201,7 @@ namespace tk { namespace dnn {
tk::dnn::Network *net = nullptr;
// layers without activations to retrieve correct id number
// layers without activations to retrive correct id number
std::vector<tk::dnn::Layer*> netLayers;
std::ifstream if_cfg(cfg_file);
@@ -278,133 +257,7 @@ namespace tk { namespace dnn {
}
return net;
}
std::vector<int> noYolosLine(const std::string &cfg_file){
std::ifstream if_cfg(cfg_file);
if(!if_cfg.is_open())
FatalError("cloud not open cfg file: " + cfg_file);
std::string line;
std::vector<int> lineNo;
int count = 0;
while(std::getline(if_cfg,line)){
std::size_t found = line.find("#");
if ( found != std::string::npos ) {
line = line.substr(0, found);
}
// skip empty lines
if(line.empty())
continue;
if(line == "[yolo]"){
lineNo.push_back(count);
}
count++;
}
return lineNo;
}
void loadYoloInfo(const std::string &cfg_file,int lineNo,std::vector<float> &mask,std::vector<float> &anchors,int &num,int &classes,float &nms_thresh,int &nms_kind,int &coords){
std::vector<float> maskTemp,anchorsTemp;
int classesTemp,numTemp,nmsKindTemp;
int new_coordsTemp=0;
float nmsThreshTemp=0.45;
std::ifstream if_cfg(cfg_file);
if(!if_cfg.is_open())
FatalError("cloud not open cfg file: " + cfg_file);
std::string line;
int count = 0;
while(std::getline(if_cfg,line)){
std::string name,value;
std::size_t found = line.find("#");
if ( found != std::string::npos ) {
line = line.substr(0, found);
}
// skip empty lines
if(line.empty())
continue;
if(count > lineNo && count <=lineNo+30){
divideNameAndValue(line,name,value);
if(name == "mask "){
maskTemp = fromStringToFloatVec(value,',');
}
if(name == "anchors "){
anchorsTemp = fromStringToFloatVec(value,',');
}
if(name == "classes"){
classesTemp = std::stoi(value);
}
if(name == "num"){
numTemp = std::stoi(value);
}
if(name == "nms_kind"){
if(value == "greedynms"){
nmsKindTemp = 0;
}else if(value == "diounms"){
nmsKindTemp=1;
}
else{
std::cout<<"NMS NOT SUPPORTED DEFAULTING TO GREEDYNMS"<<std::endl;
nmsKindTemp=0;
}
}
if(name == "new_coords"){
new_coordsTemp = std::stoi(value);
}
if(name == "beta_nms"){
nmsThreshTemp = std::stof(value);
}
}
count++;
}
mask = maskTemp;
anchors = anchorsTemp;
num = numTemp;
nms_kind = nmsKindTemp;
nms_thresh = nmsThreshTemp;
coords = new_coordsTemp;
classes = classesTemp;
}
void loadYoloInitInfo(int &channels,int &width,int &height,const std::string &cfg_file){
std::ifstream if_cfg(cfg_file);
if(!if_cfg.is_open())
FatalError("cloud not open cfg file: " + cfg_file);
std::string line;
int count = 0;
while(std::getline(if_cfg,line)){
if(count == 7){
std::string name,value;
divideNameAndValue(line,name,value);
if(name == "width"){
width = std::stoi(value);
}
}
if(count == 8){
std::string name,value;
divideNameAndValue(line,name,value);
if(name == "height"){
height = std::stoi(value);
}
}
if(count == 9){
std::string name,value;
divideNameAndValue(line,name,value);
if(name == "channels"){
channels = std::stoi(value);
break;
}
else{
std::cerr<<"EXITING PROGRAM DUE TO INSUFFICENT DATA FROM CFG"<<std::endl;
break;
}
}
count++;
}
}
}}
src/DeformConv2d.cpp
+1 -7
View File
@@ -73,12 +73,6 @@ DeformConv2d::DeformConv2d( Network *net, int out_ch, int deformable_group, int
output_dim.c = out_ch;
initCUDNN();
if(this->deformableGroup != 1)
MACC = kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
else
MACC = input_dim.c*kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
//allocate data for infer result
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
}
@@ -101,7 +95,7 @@ dnnType* DeformConv2d::infer(dataDim_t &dim, dnnType* srcData) {
// split conv2d outputs into offset and mask
checkCuda(cudaMemcpy(offset, output_conv, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
checkCuda(cudaMemcpy(mask, output_conv + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
// kernel sigmoid
// kernel sigmoide
activationSIGMOIDForward(mask, mask, chunk_dim);
// deformable convolution
src/Dense.cpp
+1 -1
View File
@@ -37,7 +37,7 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) {
// place bias into dstData
checkCuda( cudaMemcpy(dstData, bias_d, dim_y*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
//do matrix multiplication
//do matrix moltiplication
checkERROR( cublasSgemv(net->cublasHandle, CUBLAS_OP_T,
dim_x, dim_y,
&alpha,
src/Flatten.cpp
-5
View File
@@ -15,11 +15,6 @@ Flatten::Flatten(Network *net) : Layer(net) {
output_dim.w = 1;
output_dim.l = 1;
this->h = 1;
this->w = 1;
this->rows = input_dim.c;
this->cols = input_dim.h * input_dim.w;
this->c = input_dim.w * input_dim.h * input_dim.c;
}
Flatten::~Flatten() {
src/Int8BatchStream.cpp
+10 -10
View File
@@ -8,14 +8,14 @@
BatchStream::BatchStream(tk::dnn::dataDim_t dim, int batchSize, int maxBatches, const std::string& fileimglist, const std::string& filelabellist) {
mBatchSize = batchSize;
mMaxBatches = maxBatches;
mDims = nvinfer1::Dims4{ dim.n, dim.c, dim.h, dim.w };
mDims = nvinfer1::DimsNCHW{ dim.n, dim.c, dim.h, dim.w };
mHeight = dim.h;
mWidth = dim.w;
mImageSize = mDims.d[1]*mDims.d[2]*mDims.d[3];
mImageSize = mDims.c()*mDims.h()*mDims.w();
mBatch.resize(mBatchSize*mImageSize, 0);
mLabels.resize(mBatchSize, 0);
mFileBatch.resize(mDims.d[0]*mImageSize, 0);
mFileLabels.resize(mDims.d[0], 0);
mFileBatch.resize(mDims.n()*mImageSize, 0);
mFileLabels.resize(mDims.n(), 0);
mFileImgList = fileimglist;
readInListFile(fileimglist, mListImg);
mFileLabelList = filelabellist;
@@ -27,7 +27,7 @@ BatchStream::BatchStream(tk::dnn::dataDim_t dim, int batchSize, int maxBatches,
void BatchStream::reset(int firstBatch) {
mBatchCount = 0;
mFileCount = 0;
mFileBatchPos = mDims.d[0];
mFileBatchPos = mDims.n();
skip(firstBatch);
}
@@ -37,11 +37,11 @@ bool BatchStream::next() {
return false;
for (int csize = 1, batchPos = 0; batchPos < mBatchSize; batchPos += csize, mFileBatchPos += csize) {
assert(mFileBatchPos > 0 && mFileBatchPos <= mDims.d[0]);
if (mFileBatchPos == mDims.d[0] && !update())
assert(mFileBatchPos > 0 && mFileBatchPos <= mDims.n());
if (mFileBatchPos == mDims.n() && !update())
return false;
csize = std::min(mBatchSize - batchPos, mDims.d[0] - mFileBatchPos);
csize = std::min(mBatchSize - batchPos, mDims.n() - mFileBatchPos);
std::copy_n(getFileBatch() + mFileBatchPos * mImageSize, csize * mImageSize, getBatch() + batchPos * mImageSize);
std::copy_n(getFileLabels() + mFileBatchPos, csize, getLabels() + batchPos);
}
@@ -50,8 +50,8 @@ bool BatchStream::next() {
}
void BatchStream::skip(int skipCount) {
if (mBatchSize >= mDims.d[0] && mBatchSize%mDims.d[0] == 0 && mFileBatchPos == mDims.d[0]) {
mFileCount += skipCount * mBatchSize / mDims.d[0];
if (mBatchSize >= mDims.n() && mBatchSize%mDims.n() == 0 && mFileBatchPos == mDims.n()) {
mFileCount += skipCount * mBatchSize / mDims.n();
return;
}
src/Int8Calibrator.cpp
+5 -5
View File
@@ -8,13 +8,13 @@ Int8EntropyCalibrator::Int8EntropyCalibrator(BatchStream& stream, int firstBatch
mCalibTableFilePath(calibTableFilePath),
mInputBlobName(inputBlobName.c_str()),
mReadCache(readCache) {
nvinfer1::Dims4 dims = mStream.getDims();
mInputCount = mStream.getBatchSize() + dims.d[1]*dims.d[2]*dims.d[3];
nvinfer1::DimsNCHW dims = mStream.getDims();
mInputCount = mStream.getBatchSize() * dims.c() * dims.h() * dims.w();
checkCuda(cudaMalloc(&mDeviceInput, mInputCount * sizeof(float)));
mStream.reset(firstBatch);
}
bool Int8EntropyCalibrator::getBatch(void* bindings[], const char* names[], int nbBindings) NOEXCEPT {
bool Int8EntropyCalibrator::getBatch(void* bindings[], const char* names[], int nbBindings) {
if (!mStream.next())
return false;
@@ -24,7 +24,7 @@ bool Int8EntropyCalibrator::getBatch(void* bindings[], const char* names[], int
return true;
}
const void* Int8EntropyCalibrator::readCalibrationCache(size_t& length) NOEXCEPT {
const void* Int8EntropyCalibrator::readCalibrationCache(size_t& length) {
mCalibrationCache.clear();
assert(!mCalibTableFilePath.empty());
std::ifstream input(mCalibTableFilePath, std::ios::binary);
@@ -38,7 +38,7 @@ const void* Int8EntropyCalibrator::readCalibrationCache(size_t& length) NOEXCEPT
return length ? &mCalibrationCache[0] : nullptr;
}
void Int8EntropyCalibrator::writeCalibrationCache(const void* cache, size_t length) NOEXCEPT {
void Int8EntropyCalibrator::writeCalibrationCache(const void* cache, size_t length) {
assert(!mCalibTableFilePath.empty());
std::ofstream output(mCalibTableFilePath, std::ios::binary);
output.write(reinterpret_cast<const char*>(cache), length);
src/LSTM.cpp
+9 -14
View File
@@ -87,22 +87,17 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig
checkCUDNN(cudnnCreateRNNDescriptor(&rnnDesc));
#if CUDNN_MAJOR > 7
checkCUDNN(cudnnSetRNNDescriptor_v6(net->cudnnHandle,rnnDesc, stateSize, numLayers, dropoutDesc,
cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT,
//(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL),
cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL,
cudnnRNNMode_t::CUDNN_LSTM,
cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD,
net->dataType));
checkCUDNN(cudnnSetRNNDescriptor_v6(net->cudnnHandle,
#else
checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle,rnnDesc, stateSize, numLayers, dropoutDesc,
checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle,
#endif
rnnDesc, stateSize, numLayers, dropoutDesc,
cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT,
//(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL),
cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL,
cudnnRNNMode_t::CUDNN_LSTM,
cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD,
net->dataType));
#endif
// Get temp space sizes
@@ -138,7 +133,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig
output_dim = input_dim;
output_dim.c = stateSize*(bidirectional ? 2 : 1);
// if retunseq is disabled only the last timestamp is returned
// if retunseq is disabled only the last timestep is returned
if(!returnSeq) {
output_dim.h = 1;
output_dim.w = 1;
@@ -259,7 +254,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) {
rnnDesc,
seqLen, // number of time steps (nT)
x_desc_vec_.data(), // input array of desc (nT*nC_in)
srcF, // input pointer
srcF, // input pointer
hx_desc_, // initial hidden state desc
hx_ptr, // initial hidden state pointer
cx_desc_, // initial cell state desc
@@ -286,7 +281,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) {
rnnDesc,
seqLen, // number of time steps (nT)
x_desc_vec_.data(), // input array of desc (nT*nC_in)
srcB, // input pointer
srcB, // input pointer
hx_desc_, // initial hidden state desc
hx_ptr, // initial hidden state pointer
cx_desc_, // initial cell state desc
@@ -294,7 +289,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) {
w_desc_, // weights desc
wb_ptr, // weights pointer
y_desc_vec_.data(), // output desc (nT*nC_out)
dstB_NR, // output pointer
dstB_NR, // output pointer
hy_desc_, // final hidden state desc
hy_ptr, // final hidden state pointer
cy_desc_, // final cell state desc
@@ -312,7 +307,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) {
one_output_dim.c*sizeof(dnnType), cudaMemcpyDeviceToDevice));
}
// if retunseq is disabled only the last timestamp is returned
// if retunseq is disabled only the last timestep is returned
if(returnSeq) {
// forward transpose
matrixTranspose(net->cublasHandle, dstF, dstData,
src/Layer.cpp
-2
View File
@@ -18,8 +18,6 @@ Layer::Layer(Network *net) {
if(!net->addLayer(this))
FatalError("Net reached max number of layers");
}
feature_map_size = input_dim.tot() + output_dim.tot();
}
Layer::~Layer() {
src/LayerWgs.cpp
+2 -6
View File
@@ -19,8 +19,6 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
int seek = 0;
readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d, seek);
seek += inputs*outputs*kh*kw*kl;
n_params = seek;
this->additional_bias = additional_bias;
if(additional_bias) {
readBinaryFile(weights_path.c_str(), outputs, &bias2_h, &bias2_d, seek);
@@ -28,17 +26,15 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
}
readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek);
seek += outputs;
this->batchnorm = batchnorm;
if(batchnorm) {
seek += outputs;
readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek);
seek += outputs;
readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek);
seek += outputs;
readBinaryFile(weights_path.c_str(), outputs, &variance_h, &variance_d, seek);
seek += outputs;
float eps = TKDNN_BN_MIN_EPSILON;
@@ -109,7 +105,7 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
float2half(tmp_d, variance16_d, b_size);
cudaMemcpy(variance16_h, variance16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
//convert scales
//conver scales
float2half(scales_d, scales16_d, b_size);
cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
src/MobilenetDetection.cpp
+3 -4
View File
@@ -126,13 +126,12 @@ float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){
return iou;
}
bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){
bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str());
imageSize = netRT->input_dim.h;
classes = n_classes;
nBatches = n_batches;
confThreshold = conf_thresh;
SSDSpec specs[N_SSDSPEC];
@@ -198,7 +197,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe
"bottle" , "wine glass" , "cup" , "fork" , "knife" , "spoon" , "bowl" , "banana" ,
"apple" , "sandwich" , "orange" , "broccoli" , "carrot" , "hot dog" , "pizza" ,
"donut" , "cake" , "chair" , "sofa" , "pottedplant" , "bed" , "diningtable" ,
"toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" ,
"toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" ,
"cell phone" , "microwave" , "oven" , "toaster" , "sink" , "refrigerator" ,
"book" , "clock" , "vase" , "scissors" , "teddy bear" , "hair drier" , "toothbrush"};
classesNames = std::vector<std::string>(classes_names_, std::end(classes_names_));
@@ -207,7 +206,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe
else{
FatalError("Number of classes not supported for mobilenet");
}
return true;
return 1;
}
void MobilenetDetection::preprocess(cv::Mat &frame, const int bi){
src/MulAdd.cpp
+1 -1
View File
@@ -12,7 +12,7 @@ MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) {
int size = input_dim.tot();
// create a vector with all value set to add
// create a vector with all value setted to add
dnnType *add_vector_h = new dnnType[size];
for(int i=0; i<size; i++)
add_vector_h[i] = add;
src/Network.cpp
-36
View File
@@ -96,28 +96,6 @@ dataDim_t Network::getOutputDim() {
return layers[num_layers-1]->output_dim;
}
void Network::adjustFeatureMapSizeWithShortcuts(){
layerType_t layer_type;
int shortcutted_idx;
for(int i=0; i<num_layers; i++) {
layer_type = layers[i]->getLayerType();
if(layer_type == LAYER_SHORTCUT){
shortcutted_idx = -1;
for(int j=0; j<num_layers; j++) {
if(static_cast<tk::dnn::Shortcut*>(layers[i])->backLayer == layers[j]){
shortcutted_idx = j;
break;
}
}
if(shortcutted_idx == -1)
FatalError("Problem when computing featuer_map_size with shortcuts");
for(int j=shortcutted_idx+1; j<i; ++j)
layers[j]->feature_map_size += layers[shortcutted_idx]->output_dim.tot();
}
}
}
void Network::print() {
printCenteredTitle(" NETWORK MODEL ", '=', 60);
@@ -128,21 +106,10 @@ void Network::print() {
std::cout.width(16); std::cout<<std::left<<"output (H*W,CH)";
std::cout<<"\n";
adjustFeatureMapSizeWithShortcuts();
long long unsigned int tot_params = 0;
long long unsigned int max_feature_map_size = 0;
long long unsigned int tot_MACC = 0;
for(int i=0; i<num_layers; i++) {
dataDim_t in = layers[i]->input_dim;
dataDim_t out = layers[i]->output_dim;
tot_params += layers[i]->n_params;
tot_MACC += layers[i]->MACC;
if(layers[i]->feature_map_size> max_feature_map_size)
max_feature_map_size = layers[i]->feature_map_size;
std::cout.width(3); std::cout<<std::right<<i;
std::cout<<" ";
std::cout.width(16); std::cout<<std::left<<layers[i]->getLayerName();
@@ -161,9 +128,6 @@ void Network::print() {
}
printCenteredTitle("", '=', 60);
std::cout<<"\n";
std::cout<<"N params: "<<tot_params<<std::endl;
std::cout<<"Max feature map size: "<<max_feature_map_size<<std::endl;
std::cout<<"N MACC: "<<tot_MACC<<std::endl<<std::endl;
printCudaMemUsage();
}
const char *Network::getNetworkRTName(const char *network_name){
src/NetworkRT.cpp
+286 -500
View File
File diff suppressed because it is too large Load Diff
src/NetworkViz.cpp
+15 -380
View File
@@ -6,389 +6,23 @@
namespace tk { namespace dnn {
cv::Mat mapillary_15_map(cv::Mat adjMap){
// cv::imshow("test", adjMap);
// cv::waitKey(0);
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
//animal
M3.at<uchar>(0)=165;
M2.at<uchar>(0)=42;
M1.at<uchar>(0)=45;
//curb
M3.at<uchar>(1)=196;
M2.at<uchar>(1)=196;
M1.at<uchar>(1)=196;
//barrier
M3.at<uchar>(2)=90;
M2.at<uchar>(2)=120;
M1.at<uchar>(2)=150;
//road
M3.at<uchar>(3)=128;
M2.at<uchar>(3)=64;
M1.at<uchar>(3)=128;
//building
M3.at<uchar>(4)=70;
M2.at<uchar>(4)=70;
M1.at<uchar>(4)=70;
//person
M3.at<uchar>(5)=220;
M2.at<uchar>(5)=20;
M1.at<uchar>(5)=60;
//roadmark
M3.at<uchar>(6)=255;
M2.at<uchar>(6)=255;
M1.at<uchar>(6)=255;
//nature
M3.at<uchar>(7)=107;
M2.at<uchar>(7)=142;
M1.at<uchar>(7)=35;
//sky
M3.at<uchar>(8)=70;
M2.at<uchar>(8)=130;
M1.at<uchar>(8)=180;
//billboard
M3.at<uchar>(9)=220;
M2.at<uchar>(9)=220;
M1.at<uchar>(9)=220;
//pole
M3.at<uchar>(10)=153;
M2.at<uchar>(10)=153;
M1.at<uchar>(10)=153;
//traffic sign
M3.at<uchar>(11)=128;
M2.at<uchar>(11)=128;
M1.at<uchar>(11)=128;
//bike
M3.at<uchar>(12)=119;
M2.at<uchar>(12)=11;
M1.at<uchar>(12)=32;
//vehicle
M3.at<uchar>(13)=0;
M2.at<uchar>(13)=0;
M1.at<uchar>(13)=142;
//void
for(int i=14;i<256;i++)
{
M1.at<uchar>(i)=0;
M2.at<uchar>(i)=0;
M3.at<uchar>(i)=0;
}
cv::Mat r1,r2,r3;
cv::LUT(adjMap,M1,r1);
cv::LUT(adjMap,M2,r2);
cv::LUT(adjMap,M3,r3);
std::vector<cv::Mat> planes;
planes.push_back(r1);
planes.push_back(r2);
planes.push_back(r3);
cv::Mat dst;
cv::merge(planes,dst);
return dst;
}
cv::Mat berkeley_20_map(cv::Mat adjMap){
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
//road
M3.at<uchar>(0)=128;
M2.at<uchar>(0)=64;
M1.at<uchar>(0)=128;
//sidewalk
M3.at<uchar>(1)=244;
M2.at<uchar>(1)=35;
M1.at<uchar>(1)=232;
//building
M3.at<uchar>(2)=70;
M2.at<uchar>(2)=70;
M1.at<uchar>(2)=70;
//wall
M3.at<uchar>(3)=102;
M2.at<uchar>(3)=102;
M1.at<uchar>(3)=156;
//fence
M3.at<uchar>(4)=90;
M2.at<uchar>(4)=120;
M1.at<uchar>(4)=150;
//pole
M3.at<uchar>(5)=153;
M2.at<uchar>(5)=153;
M1.at<uchar>(5)=153;
//traffic light
M3.at<uchar>(6)=250;
M2.at<uchar>(6)=170;
M1.at<uchar>(6)=30;
//traffic sign
M3.at<uchar>(7)=128;
M2.at<uchar>(7)=128;
M1.at<uchar>(7)=128;
//nature
M3.at<uchar>(8)=107;
M2.at<uchar>(8)=142;
M1.at<uchar>(8)=35;
//ground
M3.at<uchar>(9)=0;
M2.at<uchar>(9)=192;
M1.at<uchar>(9)=0;
//sky
M3.at<uchar>(10)=70;
M2.at<uchar>(10)=130;
M1.at<uchar>(10)=180;
//person
M3.at<uchar>(11)=220;
M2.at<uchar>(11)=20;
M1.at<uchar>(11)=60;
//rider
M3.at<uchar>(12)=255;
M2.at<uchar>(12)=0;
M1.at<uchar>(12)=100;
//car
M3.at<uchar>(13)=0;
M2.at<uchar>(13)=0;
M1.at<uchar>(13)=142;
//truck
M3.at<uchar>(14)=0;
M2.at<uchar>(14)=0;
M1.at<uchar>(14)=70;
//bus
M3.at<uchar>(15)=0;
M2.at<uchar>(15)=60;
M1.at<uchar>(15)=100;
//train
M3.at<uchar>(16)=0;
M2.at<uchar>(16)=0;
M1.at<uchar>(16)=192;
//motorbike
M3.at<uchar>(17)=0;
M2.at<uchar>(17)=0;
M1.at<uchar>(17)=230;
//bike
M3.at<uchar>(18)=119;
M2.at<uchar>(18)=11;
M1.at<uchar>(18)=32;
//void
for(int i=19;i<256;i++)
{
M1.at<uchar>(i)=0;
M2.at<uchar>(i)=0;
M3.at<uchar>(i)=0;
}
cv::Mat r1,r2,r3;
cv::LUT(adjMap,M1,r1);
cv::LUT(adjMap,M2,r2);
cv::LUT(adjMap,M3,r3);
std::vector<cv::Mat> planes;
planes.push_back(r1);
planes.push_back(r2);
planes.push_back(r3);
cv::Mat dst;
cv::merge(planes,dst);
return dst;
}
cv::Mat cityscapes_19_map(cv::Mat adjMap){
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
//road
M3.at<uchar>(0)=128;
M2.at<uchar>(0)=64;
M1.at<uchar>(0)=128;
//sidewalk
M3.at<uchar>(1)=244;
M2.at<uchar>(1)=35;
M1.at<uchar>(1)=232;
//building
M3.at<uchar>(2)=70;
M2.at<uchar>(2)=70;
M1.at<uchar>(2)=70;
//wall
M3.at<uchar>(3)=102;
M2.at<uchar>(3)=102;
M1.at<uchar>(3)=156;
//fence
M3.at<uchar>(4)=190;
M2.at<uchar>(4)=153;
M1.at<uchar>(4)=153;
//pole
M3.at<uchar>(5)=153;
M2.at<uchar>(5)=153;
M1.at<uchar>(5)=153;
//traffic light
M3.at<uchar>(6)=250;
M2.at<uchar>(6)=170;
M1.at<uchar>(6)=30;
//traffic sign
M3.at<uchar>(7)=220;
M2.at<uchar>(7)=220;
M1.at<uchar>(7)=0;
//vegetation
M3.at<uchar>(8)=107;
M2.at<uchar>(8)=142;
M1.at<uchar>(8)=35;
//terrain
M3.at<uchar>(9)=152;
M2.at<uchar>(9)=251;
M1.at<uchar>(9)=152;
//sky
M3.at<uchar>(10)=70;
M2.at<uchar>(10)=130;
M1.at<uchar>(10)=180;
//person
M3.at<uchar>(11)=220;
M2.at<uchar>(11)=20;
M1.at<uchar>(11)=60;
//rider
M3.at<uchar>(12)=255;
M2.at<uchar>(12)=0;
M1.at<uchar>(12)=0;
//car
M3.at<uchar>(13)=0;
M2.at<uchar>(13)=0;
M1.at<uchar>(13)=142;
//truck
M3.at<uchar>(14)=0;
M2.at<uchar>(14)=0;
M1.at<uchar>(14)=70;
//bus
M3.at<uchar>(15)=0;
M2.at<uchar>(15)=60;
M1.at<uchar>(15)=100;
//train
M3.at<uchar>(16)=0;
M2.at<uchar>(16)=80;
M1.at<uchar>(16)=100;
//motorcycle
M3.at<uchar>(17)=0;
M2.at<uchar>(17)=0;
M1.at<uchar>(17)=230;
//bicycle
M3.at<uchar>(18)=119;
M2.at<uchar>(18)=11;
M1.at<uchar>(18)=32;
//void
for(int i=19;i<256;i++)
{
M1.at<uchar>(i)=0;
M2.at<uchar>(i)=0;
M3.at<uchar>(i)=0;
}
cv::Mat r1,r2,r3;
cv::LUT(adjMap,M1,r1);
cv::LUT(adjMap,M2,r2);
cv::LUT(adjMap,M3,r3);
std::vector<cv::Mat> planes;
planes.push_back(r1);
planes.push_back(r2);
planes.push_back(r3);
cv::Mat dst;
cv::merge(planes,dst);
return dst;
}
cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, int classes) {
if(min == 0 && max == 0)
cv::minMaxIdx(map, &min, &max);
cv::Mat vizFloat2colorMap(cv::Mat map) {
double min;
double max;
cv::minMaxIdx(map, &min, &max);
cv::Mat adjMap;
// expand your range to 0..255. Similar to histEq();
map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min);
//return adjMap;
cv::Mat falseColorsMap;
switch (classes)
{
case 15:
map.convertTo(adjMap,CV_8UC1);
falseColorsMap = mapillary_15_map(adjMap);
break;
case 20:
map.convertTo(adjMap,CV_8UC1);
falseColorsMap = berkeley_20_map(adjMap);
break;
case 19:
map.convertTo(adjMap,CV_8UC1);
falseColorsMap = cityscapes_19_map(adjMap);
break;
default:
// expand your range to 0..255. Similar to histEq();
map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min);
applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_PARULA);
}
applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_HOT);
return falseColorsMap;
}
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min, double max, int classes) {
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) {
dnnType *data = nullptr;
// copy to CPU
@@ -404,13 +38,14 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int i
cv::Mat grid = cv::Mat(gridSize, CV_8UC3, cv::Scalar(0));
for(int i=0; i<dim.c;i++) {
cv::Mat raw = vizFloat2colorMap(cv::Mat(cv::Size(dim.w, dim.h),CV_32FC1, data + dim.w*dim.h*i), min, max, classes);
cv::Mat raw = vizFloat2colorMap(cv::Mat(cv::Size(dim.w, dim.h),CV_32FC1, data + dim.w*dim.h*i));
int r = i / gridDim;
int c = i - r * gridDim;
raw.copyTo(grid.rowRange(r*dim.h, r*dim.h + dim.h).colRange(c*dim.w, c*dim.w + dim.w));
}
cv::Size vdim(img_w, img_h);
float ar = float(dim.w)/dim.h;
cv::Size vdim(ar*imgdim, imgdim);
cv::Mat viz;
cv::resize(grid, viz, vdim, 0, 0, 0);
@@ -424,7 +59,7 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int i
cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim) {
if(layer >= net->num_layers)
FatalError("Could not viz layer\n");
return vizData2Mat(net->layers[layer]->dstData, net->layers[layer]->output_dim, imgdim, imgdim);
return vizData2Mat(net->layers[layer]->dstData, net->layers[layer]->output_dim, imgdim);
//cv::imwrite("viz/layer" + std::to_string(layer) + ".png", viz);
//cv::imshow("layer", viz);
src/Padding.cpp
-45
View File
@@ -1,45 +0,0 @@
//
// Created by perseusdg on 03/01/22.
//
#include <iostream>
#include "Layer.h"
#include "kernels.h"
namespace tk{ namespace dnn {
Padding::Padding(Network *net, int32_t pad_h, int32_t pad_w, tkdnnPaddingMode_t padding_mode,float constant) : Layer(net) {
this->paddingH = pad_h;
this->paddingW = pad_w;
this->padding_mode = padding_mode;
output_dim.c = input_dim.c;
output_dim.n = input_dim.n;
output_dim.h = input_dim.h + 2 * (this->paddingH);
output_dim.w = input_dim.w + 2 * (this->paddingW);
if(padding_mode == tkdnnPaddingMode_t::PADDING_MODE_CONSTANT){
this->constant = constant;
}else{
this->constant = 0;
}
checkCuda(cudaMalloc(&dstData,output_dim.tot()*sizeof(dnnType)));
}
Padding::~Padding() {
checkCuda(cudaFree(dstData));
}
dnnType* Padding::infer(dataDim_t &dim, float *srcData) {
fill(dstData,output_dim.tot(),0.0);
if(padding_mode == tkdnnPaddingMode_t::PADDING_MODE_REFLECTION)
{
reflection_pad2d_out_forward(paddingH, paddingW, srcData, dstData, input_dim.h, input_dim.w, input_dim.c,
input_dim.n);
}
else if(padding_mode == tkdnnPaddingMode_t::PADDING_MODE_CONSTANT){
constant_pad2d_forward(srcData,dstData,input_dim.h,input_dim.w,output_dim.h,output_dim.w,input_dim.c,
input_dim.n,paddingH,paddingW,constant);
}
dim = output_dim;
return dstData;
}
}}
src/Pooling.cpp
-1
View File
@@ -17,7 +17,6 @@ Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW,
this->pool_mode = pool_mode;
this->paddingH = paddingH;
this->paddingW = paddingW;
this->padding = winH -1;
checkCUDNN( cudnnCreatePoolingDescriptor(&poolingDesc) );
src/Region.cpp
+2 -1
View File
@@ -16,6 +16,7 @@ Region::Region(Network *net, int classes, int coords, int num) :
this->classes = classes;
this->coords = coords;
this->num = num;
// same
output_dim.n = input_dim.n;
output_dim.c = input_dim.c;
@@ -62,7 +63,7 @@ dnnType* Region::infer(dataDim_t &dim, dnnType* srcData) {
}
/* Interpret class */
/* Intepret class */
RegionInterpret::RegionInterpret(dataDim_t input_dim, dataDim_t output_dim,
int classes, int coords, int num, float thresh, std::string fname_weights) {
src/Reshape.cpp
+1 -9
View File
@@ -8,21 +8,13 @@ namespace tk { namespace dnn {
Reshape::Reshape(Network *net, dataDim_t new_dim) : Layer(net) {
checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
this->n = new_dim.n;
this->c = new_dim.c;
this->h = new_dim.h;
this->w = new_dim.w;
output_dim.n = new_dim.n;
output_dim.c = new_dim.c;
output_dim.h = new_dim.h;
output_dim.w = new_dim.w;
output_dim.l = new_dim.l;
output_dim = new_dim;
if(input_dim.tot() != output_dim.tot())
FatalError("Reshape dimension mismatch");
}
Reshape::~Reshape() {
src/Resize.cpp
-39
View File
@@ -1,39 +0,0 @@
#include <iostream>
#include "Layer.h"
#include "kernels.h"
namespace tk { namespace dnn {
Resize::Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed, ResizeMode_t mode) : Layer(net) {
this->mode = mode;
if(fixed){
output_dim.c = scale_c;
output_dim.h = scale_h;
output_dim.w = scale_w;
}
else{
output_dim.c *= scale_c;
output_dim.h *= scale_h;
output_dim.w *= scale_w;
}
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
}
Resize::~Resize() {
checkCuda( cudaFree(dstData) );
}
dnnType* Resize::infer(dataDim_t &dim, dnnType* srcData) {
resizeForward(srcData, dstData, dim.n, dim.c, dim.h, dim.w,
output_dim.c, output_dim.h, output_dim.w);
dim = output_dim;
return dstData;
}
}}
src/Shortcut.cpp
+5 -9
View File
@@ -5,19 +5,15 @@
namespace tk { namespace dnn {
Shortcut::Shortcut(Network *net, Layer *backLayer, bool mul) : Layer(net) {
Shortcut::Shortcut(Network *net, Layer *backLayer) : Layer(net) {
this->backLayer = backLayer;
this->mul = mul;
this->c = input_dim.c;
this->h = input_dim.h;
this->w = input_dim.w;
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
if( ( backLayer->output_dim.c != input_dim.c && mul ) ||
(( backLayer->output_dim.w != input_dim.w || backLayer->output_dim.h != input_dim.h ) && !mul ) )
if( /*backLayer->output_dim.c != input_dim.c ||*/
backLayer->output_dim.w != input_dim.w ||
backLayer->output_dim.h != input_dim.h )
FatalError("Shortcut dim missmatch");
}
Shortcut::~Shortcut() {
@@ -30,7 +26,7 @@ dnnType* Shortcut::infer(dataDim_t &dim, dnnType* srcData) {
dataDim_t bdim = this->backLayer->output_dim;
checkCuda(cudaMemcpy(dstData, srcData, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1, mul);
shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1);
//update data dimensions
dim = output_dim;
src/Upsample.cpp
-3
View File
@@ -14,9 +14,6 @@ Upsample::Upsample(Network *net, int stride) : Layer(net) {
output_dim.h = input_dim.h*stride;
output_dim.w = input_dim.w*stride;
output_dim.l = input_dim.l;
this->c = input_dim.c;
this->h = input_dim.h;
this->w = input_dim.w;
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
}
src/Yolo.cpp
+18 -61
View File
@@ -9,10 +9,9 @@
#include "Layer.h"
#include "kernels.h"
namespace tk { namespace dnn {
Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_masks, float scale_xy, double nms_thresh, nmsKind_t nsm_kind, int new_coords) :
Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_masks, float scale_xy) :
Layer(net) {
this->final = true;
@@ -20,9 +19,6 @@ Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_
this->num = num;
this->n_masks = n_masks;
this->scaleXY = scale_xy;
this->nms_thresh = nms_thresh;
this->nsm_kind = nsm_kind;
this->new_coords = new_coords;
// load anchors
if(fname_weights != "") {
@@ -63,21 +59,12 @@ int entry_index(int batch, int location, int entry,
entry*input_dim.w*input_dim.h + loc;
}
Yolo::box get_yolo_box(float *x, float *biases, int n, int index, int i, int j, int lw, int lh, int w, int h, int stride, int new_coords) {
Yolo::box get_yolo_box(float *x, float *biases, int n, int index, int i, int j, int lw, int lh, int w, int h, int stride) {
Yolo::box b;
if(new_coords == 0){
b.x = (i + x[index + 0*stride]) / lw;
b.y = (j + x[index + 1*stride]) / lh;
b.w = exp(x[index + 2*stride]) * biases[2*n] / w;
b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h;
}
else{
b.x = (i + x[index + 0 * stride] ) / lw;
b.y = (j + x[index + 1 * stride] ) / lh;
b.w = x[index + 2 * stride] * x[index + 2 * stride] * 4 * biases[2 * n] / w;
b.h = x[index + 3 * stride] * x[index + 3 * stride] * 4 * biases[2 * n + 1] / h;
}
b.x = (i + x[index + 0*stride]) / lw;
b.y = (j + x[index + 1*stride]) / lh;
b.w = exp(x[index + 2*stride]) * biases[2*n] / w;
b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h;
return b;
}
@@ -88,16 +75,12 @@ dnnType* Yolo::infer(dataDim_t &dim, dnnType* srcData) {
for (int b = 0; b < dim.n; ++b){
for(int n = 0; n < n_masks; ++n){
int index = entry_index(b, n*dim.w*dim.h, 0, classes, input_dim, output_dim);
if (new_coords == 1){
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
}
else{
activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h);
activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h);
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim);
activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h);
}
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim);
activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h);
}
}
@@ -133,7 +116,7 @@ void correct_yolo_boxes(Yolo::detection *dets, int n, int w, int h, int netw, in
}
}
int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh, int newCoords) {
int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh) {
if(predictions == nullptr)
predictions = new dnnType[output_dim.tot()];
@@ -157,7 +140,7 @@ int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int net
if(objectness <= thresh) continue;
int box_index = entry_index(0, n*lw*lh + i, 0, classes, input_dim, output_dim);
dets[count].bbox = get_yolo_box(predictions, bias_h, mask_h[n], box_index, col, row, lw, lh, netw, neth, lw*lh, newCoords);
dets[count].bbox = get_yolo_box(predictions, bias_h, mask_h[n], box_index, col, row, lw, lh, netw, neth, lw*lh);
dets[count].objectness = objectness;
dets[count].classes = classes;
for(j = 0; j < classes; ++j){
@@ -210,32 +193,6 @@ float yolo_box_iou(Yolo::box a, Yolo::box b)
return yolo_box_intersection(a, b)/yolo_box_union(a, b);
}
void box_c(const Yolo::box a, const Yolo::box b, float& top, float& bot, float& left, float& right) {
top = (std::min)(a.y - a.h / 2, b.y - b.h / 2);
bot = (std::max)(a.y + a.h / 2, b.y + b.h / 2);
left = (std::min)(a.x - a.w / 2, b.x - b.w / 2);
right = (std::max)(a.x + a.w / 2, b.x + b.w / 2);
}
// https://github.com/Zzh-tju/DIoU-darknet
// https://arxiv.org/abs/1911.08287
float yolo_box_diou(const Yolo::box a, const Yolo::box b, const float nms_thresh=0.6)
{
float top, bot, left, right;
box_c(a, b, top, bot, left, right);
float w = right - left;
float h = bot - top;
float c = w * w + h * h;
float iou = yolo_box_iou(a, b);
if (c == 0)
return iou;
float d = (a.x - b.x) * (a.x - b.x) + (a.y - b.y) * (a.y - b.y);
float u = pow(d / c, nms_thresh);
float diou_term = u;
return iou - diou_term;
}
int yolo_nms_comparator(const void *pa, const void *pb)
{
Yolo::detection a = *(Yolo::detection *)pa;
@@ -262,7 +219,8 @@ Yolo::detection *Yolo::allocateDetections(int nboxes, int classes) {
return dets;
}
void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes, double nms_thresh, nmsKind_t nsm_kind) {
void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes) {
double nms_thresh = 0.45;
int total = ndets;
int i, j, k;
@@ -278,7 +236,6 @@ void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes, double
}
total = k+1;
float thresh = 0.45f;
for(k = 0; k < classes; ++k){
for(i = 0; i < total; ++i){
dets[i].sort_class = k;
@@ -289,13 +246,13 @@ void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes, double
box a = dets[i].bbox;
for(j = i+1; j < total; ++j){
box b = dets[j].bbox;
if (nsm_kind == GREEDY_NMS && yolo_box_iou(a, b) > thresh)
dets[j].prob[k] = 0;
else if (nsm_kind == DIOU_NMS && yolo_box_diou(a, b, nms_thresh) > thresh)
if (yolo_box_iou(a, b) > nms_thresh){
dets[j].prob[k] = 0;
}
}
}
}
}
}}
src/Yolo3Detection.cpp
+119 -53
View File
@@ -3,23 +3,22 @@
namespace tk { namespace dnn {
bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) {
bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches) {
//convert network to tensorRT
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(nullptr, (tensor_path).c_str() );
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
nBatches = n_batches;
confThreshold = conf_thresh;
tk::dnn::dataDim_t idim = netRT->input_dim;
idim.n = nBatches;
if(netRT->yolo_plugins.size() < 2 ) {
if(netRT->pluginFactory->n_yolos < 2 ) {
FatalError("this is not yolo3");
}
for(int i=0; i<netRT->yolo_plugins.size(); i++) {
nvinfer1::YoloRT *yRT = netRT->yolo_plugins[i];
for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
YoloRT *yRT = netRT->pluginFactory->yolos[i];
classes = yRT->classes;
num = yRT->num;
nMasks = yRT->n_masks;
@@ -28,13 +27,10 @@ namespace tk { namespace dnn {
yolo[i] = new tk::dnn::Yolo(nullptr, classes, nMasks, ""); // yolo without input and bias
yolo[i]->mask_h = new dnnType[nMasks];
yolo[i]->bias_h = new dnnType[num*nMasks*2];
memcpy(yolo[i]->mask_h, yRT->mask.data(), sizeof(dnnType)*nMasks);
memcpy(yolo[i]->bias_h, yRT->bias.data(), sizeof(dnnType)*num*nMasks*2);
memcpy(yolo[i]->mask_h, yRT->mask, sizeof(dnnType)*nMasks);
memcpy(yolo[i]->bias_h, yRT->bias, sizeof(dnnType)*num*nMasks*2);
yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, yRT->c, yRT->h, yRT->w);
yolo[i]->classesNames = yRT->classesNames;
yolo[i]->nms_thresh = yRT->nms_thresh;
yolo[i]->nsm_kind = (tk::dnn::Yolo::nmsKind_t) yRT->nms_kind;
yolo[i]->new_coords = yRT->new_coords;
}
dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
@@ -56,28 +52,80 @@ namespace tk { namespace dnn {
return true;
}
void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat orig_img, img_resized;
orig_img = cv::cuda::GpuMat(frame);
cv::cuda::resize(orig_img, img_resized, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
img_resized.convertTo(imagePreproc, CV_32FC3, 1/255.0);
//split channels
cv::cuda::split(imagePreproc,bgr);//split source
//write channels
for(int i=0; i<netRT->input_dim.c; i++) {
int size = imagePreproc.rows * imagePreproc.cols;
int ch = netRT->input_dim.c-1 -i;
bgr[ch].download(bgr_h); //TODO: don't copy back on CPU
checkCuda( cudaMemcpy(input_d + i*size + netRT->input_dim.tot()*bi, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice));
cv::Mat resize_image(cv::Mat im, int w, int h)
{
cv::Mat resized = cv::Mat(cv::Size(w,h), CV_32FC3, cv::Scalar(0) );
cv::Mat part = cv::Mat(cv::Size(w,im.rows), CV_32FC3, cv::Scalar(0) );
int r, c, k;
float w_scale = (float)(im.cols - 1) / (w - 1);
float h_scale = (float)(im.rows - 1) / (h - 1);
for(k = 0; k < im.channels(); ++k){
for(r = 0; r < im.rows; ++r){
for(c = 0; c < w; ++c){
float val = 0;
if(c == w-1 || im.cols == 1){
val = im.at<cv::Vec3f>(r, im.cols-1)[k];
} else {
float sx = c*w_scale;
int ix = (int) sx;
float dx = sx - ix;
val = (1 - dx) * im.at<cv::Vec3f>(r, ix)[k] + dx * im.at<cv::Vec3f>(r,ix+1)[k];
}
part.at<cv::Vec3f>(r,c)[k] = val;
}
}
}
#else
cv::resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
for(k = 0; k < im.channels(); ++k){
for(r = 0; r < h; ++r){
float sy = r*h_scale;
int iy = (int) sy;
float dy = sy - iy;
for(c = 0; c < w; ++c){
float val = (1-dy) * part.at<cv::Vec3f>(iy, c)[k];
resized.at<cv::Vec3f>(r, c)[k] = val;
}
if(r == h-1 || im.rows == 1) continue;
for(c = 0; c < w; ++c){
float val = dy * part.at<cv::Vec3f>(iy+1, c)[k];
resized.at<cv::Vec3f>(r,c)[k] += val;
}
}
}
return resized;
}
void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
frame.convertTo(imagePreproc, CV_32FC3, 1/255.0);
if(letterbox){
int im_w = frame.cols;
int im_h = frame.rows;
int net_w = netRT->input_dim.w;
int net_h = netRT->input_dim.h;
if(net_w == net_h && letterbox){
float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
int new_h = im_h/ratio;
int new_w = im_w/ratio;
imagePreproc = resize_image(imagePreproc, new_w, new_h);
cv::Mat borders;
int top = (net_h - new_h)/2;
int bottom = (net_h - new_h) - top;
int left = (net_w - new_w)/2;
int right = (net_w - new_w) - left;
cv::copyMakeBorder(imagePreproc,imagePreproc, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0.5,0.5,0.5));
}
else
FatalError("letterbox not spported with h!=w");
}
else
imagePreproc = resize_image(imagePreproc, netRT->input_dim.w, netRT->input_dim.h);
//split channels
cv::split(imagePreproc,bgr);//split source
@@ -88,41 +136,65 @@ void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
memcpy((void*)&input[idx + netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d + netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
#endif
}
void Yolo3Detection::postprocess(const int bi, const bool mAP){
//get yolo outputs
if(netRT->yolo_plugins.size() < 2){
FatalError("YOLOS WRONG!!");
}
std::vector<float *> rt_out;
//dnnType *rt_out[netRT->pluginFactory->n_yolos];
for(int i=0; i<netRT->yolo_plugins.size(); i++)
rt_out.push_back((dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi);
dnnType *rt_out[netRT->pluginFactory->n_yolos];
for(int i=0; i<netRT->pluginFactory->n_yolos; i++)
rt_out[i] = (dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi;
float x_ratio = float(originalSize[bi].width) / float(netRT->input_dim.w);
float y_ratio = float(originalSize[bi].height) / float(netRT->input_dim.h);
// compute dets
nDets = 0;
for(int i=0; i<netRT->yolo_plugins.size(); i++) {
for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
yolo[i]->dstData = rt_out[i];
yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold, yolo[i]->new_coords);
yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold);
}
tk::dnn::Yolo::mergeDetections(dets, nDets, classes, yolo[0]->nms_thresh, yolo[0]->nsm_kind);
tk::dnn::Yolo::mergeDetections(dets, nDets, classes);
int im_w = originalSize[bi].width;
int im_h = originalSize[bi].height;
int net_w = netRT->input_dim.w;
int net_h = netRT->input_dim.h;
int new_h, new_w;
int top = 0, left = 0;
if(letterbox){
float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
x_ratio = ratio;
y_ratio = ratio;
std::cout<<ratio<<std::endl;
int new_h = im_h/ratio;
int new_w = im_w/ratio;
top = (net_h - new_h)/2;
left = (net_w - new_w)/2;
}
else{
new_h = net_h;
new_w = net_w;
}
float deltaw = net_w - new_w;
float deltah = net_h - new_h;
float ratiow = (float)new_w / net_w;
float ratioh = (float)new_h / net_h;
// fill detected
detected.clear();
for(int j=0; j<nDets; j++) {
tk::dnn::Yolo::box b = dets[j].bbox;
float x0 = (b.x-b.w/2.);
float x1 = (b.x+b.w/2.);
float y0 = (b.y-b.h/2.);
float y1 = (b.y+b.h/2.);
float x0 = (b.x - left - b.w/2.);
float x1 = (b.x - left + b.w/2.);
float y0 = (b.y - top - b.h/2.);
float y1 = (b.y - top + b.h/2.);
// convert to image coords
x0 = x_ratio*x0;
@@ -143,15 +215,9 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
res.w = x1 - x0;
res.h = y1 - y0;
// FIXME: this shuld be useless
// if(mAP)
// for(int c=0; c<classes; c++)
// res.probs.push_back(dets[j].prob[c]);
detected.push_back(res);
}
}
}
batchDetected.push_back(detected);
}

Some files were not shown because too many files have changed in this diff Show More