Merge branch 'ceccocats:master' into master
This commit is contained in:
Harshvardhan Chandirasekar
GitHub
@@ -53,6 +53,9 @@ include_directories(${EIGEN3_INCLUDE_DIR})
|
||||
|
||||
find_package(OpenCV REQUIRED)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV")
|
||||
# if(OpenCV_CUDA_VERSION)
|
||||
# add_compile_definitions(OPENCV_CUDACONTRIB)
|
||||
# endif()
|
||||
|
||||
# gives problems in cross-compiling, probably malformed cmake config
|
||||
find_package(yaml-cpp REQUIRED)
|
||||
@@ -119,6 +122,14 @@ target_link_libraries(test_resnet101_cnet tkDNN)
|
||||
add_executable(test_dla34_cnet tests/centernet/dla34_cnet/dla34_cnet.cpp)
|
||||
target_link_libraries(test_dla34_cnet tkDNN)
|
||||
|
||||
add_executable(test_dla34_cnet3d tests/centernet/dla34_cnet3d/dla34_cnet3d.cpp)
|
||||
target_link_libraries(test_dla34_cnet3d tkDNN)
|
||||
|
||||
# CENTERTRACK
|
||||
|
||||
add_executable(test_dla34_ctrack tests/centertrack/dla34_ctrack/dla34_ctrack.cpp)
|
||||
target_link_libraries(test_dla34_ctrack tkDNN)
|
||||
|
||||
# SHELFNET
|
||||
add_executable(test_shelfnet tests/shelfnet/shelfnet.cpp)
|
||||
target_link_libraries(test_shelfnet tkDNN)
|
||||
@@ -139,6 +150,12 @@ target_link_libraries(map_demo tkDNN)
|
||||
add_executable(demo demo/demo/demo.cpp)
|
||||
target_link_libraries(demo tkDNN)
|
||||
|
||||
add_executable(demo3D demo/demo/demo3D.cpp)
|
||||
target_link_libraries(demo3D tkDNN)
|
||||
|
||||
add_executable(demoTracker demo/demo/demoTracker.cpp)
|
||||
target_link_libraries(demoTracker tkDNN)
|
||||
|
||||
add_executable(seg_demo demo/demo/seg_demo.cpp)
|
||||
target_link_libraries(seg_demo tkDNN)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ If you use tkDNN in your research, please cite the [following paper](https://iee
|
||||
|
||||
### What's new (20 July 2021)
|
||||
- [x] Support to sematic segmentation [README](docs/README_seg.md)
|
||||
- [x] Support 2D/3D Object Detection and Tracking [README](docs/README_2d3dtracking.md)
|
||||
- [ ] Support to TensorRT8 (WIP)
|
||||
|
||||
## FPS Results
|
||||
@@ -70,32 +71,18 @@ Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001
|
||||
- [tkDNN](#tkdnn)
|
||||
- [Index](#index)
|
||||
- [Dependencies](#dependencies)
|
||||
- [About OpenCV](#about-opencv)
|
||||
- [How to compile this repo](#how-to-compile-this-repo)
|
||||
- [Workflow](#workflow)
|
||||
- [How to export weights](#how-to-export-weights)
|
||||
- [1)Export weights from darknet](#1export-weights-from-darknet)
|
||||
- [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101)
|
||||
- [3)Export weights for CenterNet](#3export-weights-for-centernet)
|
||||
- [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd)
|
||||
- [Run the demo](#run-the-demo)
|
||||
- [FP16 inference](#fp16-inference)
|
||||
- [INT8 inference](#int8-inference)
|
||||
- [mAP demo](#map-demo)
|
||||
- [Exporting weights](#exporting-weights)
|
||||
- [Run the demos](#run-the-demos)
|
||||
- [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows-10-experimental)
|
||||
- [Existing tests and supported networks](#existing-tests-and-supported-networks)
|
||||
- [References](#references)
|
||||
- [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows-10-experimental)
|
||||
- [Dependencies-Windows](#dependencies-windows)
|
||||
- [Compiling tkDNN on Windows](#compiling-tkdnn-on-windows)
|
||||
- [Run the demo on Windows](#run-the-demo-on-windows)
|
||||
- [FP16 inference windows](#fp16-inference-windows)
|
||||
- [INT8 inference windows](#int8-inference-windows)
|
||||
- [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows)
|
||||
|
||||
|
||||
## Dependencies
|
||||
This branch works on every NVIDIA GPU that supports the following (latest tested) dependencies:
|
||||
* CUDA 11.0 (or >= 10)
|
||||
* CUDA 11.0 (or >= 10) [the segmentation only works with CUDA 10 for now]
|
||||
* cuDNN 8.0.4 (or >= 7.3)
|
||||
* TensorRT 7.2.0 (or >=5)
|
||||
* OpenCV 4.5.2 (or >=4)
|
||||
@@ -109,7 +96,7 @@ sudo apt install libyaml-cpp-dev curl libeigen3-dev
|
||||
|
||||
```
|
||||
|
||||
## About OpenCV
|
||||
#### About OpenCV
|
||||
To compile and install OpenCV4 with contrib us the script ```install_OpenCV4.sh```. It will download and compile OpenCV in Download folder.
|
||||
```
|
||||
bash scripts/install_OpenCV4.sh
|
||||
@@ -135,214 +122,23 @@ Steps needed to do inference on tkDNN with a custom neural network.
|
||||
* Create a new test and define the network, layer by layer using the weights extracted and the output to check the results.
|
||||
* Do inference.
|
||||
|
||||
## How to export weights
|
||||
## Exporting weights
|
||||
|
||||
Weights are essential for any network to run inference. For each test a folder organized as follow is needed (in the build folder):
|
||||
```
|
||||
test_nn
|
||||
|---- layers/ (folder containing a binary file for each layer with the corresponding wieghts and bias)
|
||||
|---- debug/ (folder containing a binary file for each layer with the corresponding outputs)
|
||||
```
|
||||
Therefore, once the weights have been exported, the folders layers and debug should be placed in the corresponding test.
|
||||
For specific details on how to export weights see [HERE](./docs/exporting_weights.md).
|
||||
|
||||
### 1)Export weights from darknet
|
||||
To export weights for NNs that are defined in darknet framework, use [this](https://git.hipert.unimore.it/fgatti/darknet.git) fork of darknet and follow these steps to obtain a correct debug and layers folder, ready for tkDNN.
|
||||
|
||||
```
|
||||
git clone https://git.hipert.unimore.it/fgatti/darknet.git
|
||||
cd darknet
|
||||
make
|
||||
mkdir layers debug
|
||||
./darknet export <path-to-cfg-file> <path-to-weights> layers
|
||||
```
|
||||
N.b. Use compilation with CPU (leave GPU=0 in Makefile) if you also want debug.
|
||||
|
||||
### 2)Export weights for DLA34 and ResNet101
|
||||
To get weights and outputs needed to run the tests dla34 and resnet101 use the Python script and the Anaconda environment included in the repository.
|
||||
|
||||
Create Anaconda environment and activate it:
|
||||
```
|
||||
conda env create -f file_name.yml
|
||||
source activate env_name
|
||||
python <script name>
|
||||
```
|
||||
### 3)Export weights for CenterNet
|
||||
To get the weights needed to run Centernet tests use [this](https://github.com/sapienzadavide/CenterNet.git) fork of the original Centernet.
|
||||
```
|
||||
git clone https://github.com/sapienzadavide/CenterNet.git
|
||||
```
|
||||
* follow the instruction in the README.md and INSTALL.md
|
||||
|
||||
```
|
||||
python demo.py --input_res 512 --arch resdcn_101 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_resdcn101.pth --exp_wo --exp_wo_dim 512
|
||||
python demo.py --input_res 512 --arch dla_34 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_dla_2x.pth --exp_wo --exp_wo_dim 512
|
||||
```
|
||||
### 4)Export weights for MobileNetSSD
|
||||
To get the weights needed to run Mobilenet tests use [this](https://github.com/mive93/pytorch-ssd) fork of a Pytorch implementation of SSD network.
|
||||
|
||||
```
|
||||
git clone https://github.com/mive93/pytorch-ssd
|
||||
cd pytorch-ssd
|
||||
conda env create -f env_mobv2ssd.yml
|
||||
python run_ssd_live_demo.py mb2-ssd-lite <pth-model-fil> <labels-file>
|
||||
```
|
||||
|
||||
## Darknet Parser
|
||||
tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*:
|
||||
```
|
||||
// example of parsing yolo4
|
||||
tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names");
|
||||
net->print();
|
||||
```
|
||||
All models from darknet are now parsed directly from cfg, you still need to export the weights with the described tools in the previous section.
|
||||
<details>
|
||||
<summary>Supported layers</summary>
|
||||
convolutional
|
||||
maxpool
|
||||
avgpool
|
||||
shortcut
|
||||
upsample
|
||||
route
|
||||
reorg
|
||||
region
|
||||
yolo
|
||||
</details>
|
||||
<details>
|
||||
<summary>Supported activations</summary>
|
||||
relu
|
||||
leaky
|
||||
mish
|
||||
logistic
|
||||
</details>
|
||||
|
||||
## Run the demo
|
||||
This is an example using yolov4.
|
||||
|
||||
To run the an object detection first create the .rt file by running:
|
||||
```
|
||||
rm yolo4_fp32.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_yolo4 # run the yolo test (is slow)
|
||||
```
|
||||
If you get problems in the creation, try to check the error activating the debug of TensorRT in this way:
|
||||
```
|
||||
cmake .. -DDEBUG=True
|
||||
make
|
||||
```
|
||||
|
||||
Once you have successfully created your rt file, run the demo:
|
||||
```
|
||||
./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y
|
||||
```
|
||||
In general the demo program takes 7 parameters:
|
||||
```
|
||||
./demo <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh>
|
||||
```
|
||||
where
|
||||
* ```<network-rt-file>``` is the rt file generated by a test
|
||||
* ```<<path-to-video>``` is the path to a video file or a camera input
|
||||
* ```<kind-of-network>``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
|
||||
* ```<number-of-classes>```is the number of classes the network is trained on
|
||||
* ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
|
||||
* ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
|
||||
* ```<conf-thresh>``` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed.
|
||||
|
||||
N.b. By default it is used FP32 inference
|
||||
## Run the demos
|
||||
|
||||
For specific details on how to run:
|
||||
- 2D object detection demos, details on FP16, INT8 and batching see [HERE](./docs/demo.md).
|
||||
- segmentation demos see [HERE](./docs/README_seg.md).
|
||||
- 2D/3D object detection and tracking demos see [HERE](./docs/README_2d3dtracking.md).
|
||||
- mAP demo to evaluate 2D object detectors see [HERE](./docs/mAP_demo.md).
|
||||
|
||||
|
||||
|
||||
### FP16 inference
|
||||
## tkDNN on Windows 10 (experimental)
|
||||
|
||||
To run the an object detection demo with FP16 inference follow these steps (example with yolov3):
|
||||
```
|
||||
export TKDNN_MODE=FP16 # set the half floating point optimization
|
||||
rm yolo3_fp16.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_yolo3 # run the yolo test (is slow)
|
||||
./demo yolo3_fp16.rt ../demo/yolo_test.mp4 y
|
||||
```
|
||||
N.b. Using FP16 inference will lead to some errors in the results (first or second decimal).
|
||||
|
||||
### INT8 inference
|
||||
|
||||
To run the an object detection demo with INT8 inference three environment variables need to be set:
|
||||
* ```export TKDNN_MODE=INT8```: set the 8-bit integer optimization
|
||||
* ```export TKDNN_CALIB_IMG_PATH=/path/to/calibration/image_list.txt``` : image_list.txt has in each line the absolute path to a calibration image
|
||||
* ```export TKDNN_CALIB_LABEL_PATH=/path/to/calibration/label_list.txt```: label_list.txt has in each line the absolute path to a calibration label
|
||||
|
||||
You should provide image_list.txt and label_list.txt, using training images. However, if you want to quickly test the INT8 inference you can run (from this repo root folder)
|
||||
```
|
||||
bash scripts/download_validation.sh COCO
|
||||
```
|
||||
to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD instead of COCO to download BDD validation.
|
||||
|
||||
Then a complete example using yolo3 and COCO dataset would be:
|
||||
```
|
||||
export TKDNN_MODE=INT8
|
||||
export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
|
||||
export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
|
||||
rm yolo3_int8.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_yolo3 # run the yolo test (is slow)
|
||||
./demo yolo3_int8.rt ../demo/yolo_test.mp4 y
|
||||
```
|
||||
N.B.
|
||||
* Using INT8 inference will lead to some errors in the results.
|
||||
* The test will be slower: this is due to the INT8 calibration, which may take some time to complete.
|
||||
* INT8 calibration requires TensorRT version greater than or equal to 6.0
|
||||
* Only 100 images are used to create the calibration table by default (set in the code).
|
||||
|
||||
### BatchSize bigger than 1
|
||||
```
|
||||
export TKDNN_BATCHSIZE=2
|
||||
# build tensorRT files
|
||||
```
|
||||
This will create a TensorRT file with the desired **max** batch size.
|
||||
The test will still run with a batch of 1, but the created tensorRT can manage the desired batch size.
|
||||
|
||||
### Test batch Inference
|
||||
This will test the network with random input and check if the output of each batch is the same.
|
||||
```
|
||||
./test_rtinference <network-rt-file> <number-of-batches>
|
||||
# <number-of-batches> should be less or equal to the max batch size of the <network-rt-file>
|
||||
|
||||
# example
|
||||
export TKDNN_BATCHSIZE=4 # set max batch size
|
||||
rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_yolo3 # build RT file
|
||||
./test_rtinference yolo3_fp32.rt 4 # test with a batch size of 4
|
||||
```
|
||||
|
||||
## mAP demo
|
||||
|
||||
To compute mAP, precision, recall and f1score, run the map_demo.
|
||||
|
||||
A validation set is needed.
|
||||
To download COCO_val2017 (80 classes) run (form the root folder):
|
||||
```
|
||||
bash scripts/download_validation.sh COCO
|
||||
```
|
||||
To download Berkeley_val (10 classes) run (form the root folder):
|
||||
```
|
||||
bash scripts/download_validation.sh BDD
|
||||
```
|
||||
|
||||
To compute the map, the following parameters are needed:
|
||||
```
|
||||
./map_demo <network rt> <network type [y|c|m]> <labels file path> <config file path>
|
||||
```
|
||||
where
|
||||
* ```<network rt>```: rt file of a chosen network on which compute the mAP.
|
||||
* ```<network type [y|c|m]>```: type of network. Right now only y(yolo), c(centernet) and m(mobilenet) are allowed
|
||||
* ```<labels file path>```: path to a text file containing all the paths of the ground-truth labels. It is important that all the labels of the ground-truth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the ground-truth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg.
|
||||
* ```<config file path>```: path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
cd build
|
||||
./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml
|
||||
```
|
||||
|
||||
This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to submit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate).
|
||||
For specific details on how to run tkDNN on Windows 10 see [HERE](./docs/windows.md).
|
||||
|
||||
## Existing tests and supported networks
|
||||
|
||||
@@ -369,98 +165,19 @@ This demo also creates a json file named ```net_name_COCO_res.json``` containing
|
||||
| resnet101_cnet | Centernet (Resnet101 backend)<sup>4</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download) |
|
||||
| csresnext50-panet-spp | Cross Stage Partial Network <sup>7</sup> | [COCO 2014](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download) |
|
||||
| yolo4 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
|
||||
| yolo4_320 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 320x320 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
|
||||
| yolo4_512 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
|
||||
| yolo4_608 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 608x608 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
|
||||
| yolo4_berkeley | Yolov4 <sup>8</sup> | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 540x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) |
|
||||
| yolo4tiny | Yolov4 tiny <sup>9</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) |
|
||||
| yolo4x | Yolov4x-mish <sup>9</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 640x640 | [weights](https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download) |
|
||||
| yolo4x | Yolov4x-mish <sup>9</sup> | [COCO 2017](http://cocodataset.org/) |
|
||||
| yolo4tiny_512 | Yolov4 tiny <sup>9</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) |
|
||||
80 | 640x640 | [weights](https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download) |
|
||||
| yolo4x-cps | Scaled Yolov4 <sup>10</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/AfzHE4BfTeEm2gH/download) |
|
||||
|
||||
### tkDNN on Windows 10 (experimental)
|
||||
|
||||
### Dependencies-Windows
|
||||
This branch should work on every NVIDIA GPU supported in windows with the following dependencies:
|
||||
|
||||
* WINDOWS 10 1803 or HIGHER
|
||||
* CUDA 10.0 (Recommended CUDA 11.2 )
|
||||
* CUDNN 7.6 (Recommended CUDNN 8.1.1 )
|
||||
* TENSORRT 6.0.1 (Recommended TENSORRT 7.2.3.4 )
|
||||
* OPENCV 3.4 (Recommended OPENCV 4.2.0 )
|
||||
* MSVC 16.7
|
||||
* YAML-CPP
|
||||
* EIGEN3
|
||||
* 7ZIP (ADD TO PATH)
|
||||
* NINJA 1.10
|
||||
|
||||
|
||||
All the above mentioned dependencies except 7ZIP can be installed using Microsoft's [VCPKG](https://github.com/microsoft/vcpkg.git) .
|
||||
After bootstrapping VCPKG the dependencies can be built and installed using the following command :
|
||||
|
||||
```
|
||||
opencv4(normal) - vcpkg.exe install opencv4[tbb,jpeg,tiff,opengl,openmp,png,ffmpeg,eigen]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build
|
||||
|
||||
opencv4(cuda) - vcpkg.exe install opencv4[cuda,nonfree,contrib,eigen,tbb,jpeg,tiff,opengl,openmp,png,ffmpeg]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build
|
||||
```
|
||||
To build opencv4 with cuda and cudnn version corresponding to your cuda version,vcpkg's cudnn portfile needs to be modified by adding ```$ENV{CUDA_PATH}``` at lines 16 and 17 in the portfile.cmake
|
||||
|
||||
After VCPKG finishes building and installing all the packages delete C:\temp_vcpkg_build and add C:\opt\x64-windows\bin and C:\opt\x64-windows\debug\bin to path
|
||||
|
||||
### Compiling tkDNN on Windows
|
||||
|
||||
tkDNN is built with cmake(3.15+) on windows along with ninja.Msbuild and NMake Makefiles are drastically slower when compiling the library compared to windows
|
||||
```
|
||||
git clone https://github.com/ceccocats/tkDNN.git
|
||||
cd tkdnn-windows
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release -G"Ninja" ..
|
||||
ninja -j4
|
||||
```
|
||||
|
||||
### Run the demo on Windows
|
||||
|
||||
This example uses yolo4_tiny.\
|
||||
To run the object detection file create .rt file bu running:
|
||||
```
|
||||
.\test_yolo4tiny.exe
|
||||
```
|
||||
|
||||
Once the rt file has been successfully create,run the demo using the following command:
|
||||
```
|
||||
.\demo.exe yolo4tiny_fp32.rt ..\demo\yolo_test.mp4 y
|
||||
```
|
||||
For general info on more demo paramters,check Run the demo section on top
|
||||
To run the test_all_tests.sh on windows,use git bash or msys2
|
||||
|
||||
### FP16 inference windows
|
||||
|
||||
This is an untested feature on windows.To run the object detection demo with FP16 interference follow the below steps(example with yolo4tiny):
|
||||
```
|
||||
set TKDNN_MODE=FP16
|
||||
del /f yolo4tiny_fp16.rt
|
||||
.\test_yolo4tiny.exe
|
||||
.\demo.exe yolo4tiny_fp16.rt ..\demo\yolo_test.mp4
|
||||
```
|
||||
|
||||
### INT8 inference windows
|
||||
To run object detection demo with INT8 (example with yolo4tiny):
|
||||
```
|
||||
set TKDNN_MODE=INT8
|
||||
set TKDNN_CALIB_LABEL_PATH=..\demo\COCO_val2017\all_labels.txt
|
||||
set TKDNN_CALIB_IMG_PATH=..\demo\COCO_val2017\all_images.txt
|
||||
del /f yolo4tiny_int8.rt # be sure to delete(or move) old tensorRT files
|
||||
.\test_yolo4tiny.exe # run the yolo test (is slow)
|
||||
.\demo.exe yolo4tiny_int8.rt ..\demo\yolo_test.mp4 y
|
||||
|
||||
```
|
||||
|
||||
### Known issues with tkDNN on Windows
|
||||
|
||||
Mobilenet and Centernet demos work properly only when built with msvc 16.7 in Release Mode,when built in debug mode for the mentioned networks one might encounter opencv assert errors
|
||||
|
||||
All Darknet models work properly with demo using MSVC version(16.7-16.9)
|
||||
|
||||
It is recommended to use Nvidia Driver(465+),Cuda unknown errors have been observed when using older drivers on pascal(SM 61) devices.
|
||||
|
||||
|
||||
| shelfnet | ShelfNet18_realtime<sup>11</sup> | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) |
|
||||
| shelfnet_berkeley | ShelfNet18_realtime<sup>11</sup> | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) |
|
||||
| dla34_cnet3d | Centernet3D (DLA34 backend)<sup>4</sup> | [KITTI 2017](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d) | 1 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/2MDyWGzQsTKMjmR/download) |
|
||||
| dla34_ctrack | CenterTrack (DLA34 backend)<sup>12</sup> | [NuScenes 3D](https://www.nuscenes.org/) | 7 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download) |
|
||||
|
||||
|
||||
## References
|
||||
@@ -475,3 +192,5 @@ It is recommended to use Nvidia Driver(465+),Cuda unknown errors have been obser
|
||||
8. Bochkovskiy, Alexey, Chien-Yao Wang, and Hong-Yuan Mark Liao. "YOLOv4: Optimal Speed and Accuracy of Object Detection." arXiv preprint arXiv:2004.10934 (2020).
|
||||
9. Bochkovskiy, Alexey, "Yolo v4, v3 and v2 for Windows and Linux" (https://github.com/AlexeyAB/darknet)
|
||||
10. Wang, Chien-Yao, Alexey Bochkovskiy, and Hong-Yuan Mark Liao. "Scaled-YOLOv4: Scaling Cross Stage Partial Network." arXiv preprint arXiv:2011.08036 (2020).
|
||||
11. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019.
|
||||
12. Zhou, Xingyi, Vladlen Koltun, and Philipp Krähenbühl. "Tracking objects as points." European Conference on Computer Vision. Springer, Cham, 2020.
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
#include <iostream>
|
||||
#include <signal.h>
|
||||
#include <stdlib.h> /* srand, rand */
|
||||
//#include <unistd.h>
|
||||
#include <mutex>
|
||||
|
||||
#include "demo_utils.h"
|
||||
#include "CenternetDetection3D.h"
|
||||
|
||||
bool gRun;
|
||||
bool SAVE_RESULT = false;
|
||||
|
||||
void sig_handler(int signo) {
|
||||
std::cout<<"request gateway stop\n";
|
||||
gRun = false;
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
|
||||
std::cout<<"detection\n";
|
||||
signal(SIGINT, sig_handler);
|
||||
|
||||
|
||||
std::string net = "dla34_cnet3d_fp32.rt";
|
||||
if(argc > 1)
|
||||
net = argv[1];
|
||||
#ifdef __linux__
|
||||
std::string input = "../demo/yolo_test.mp4";
|
||||
#elif _WIN32
|
||||
std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
|
||||
#endif
|
||||
|
||||
if(argc > 2)
|
||||
input = argv[2];
|
||||
std::string calib_params = "";
|
||||
if(argc > 3)
|
||||
calib_params = argv[3];
|
||||
char ntype = 'c';
|
||||
if(argc > 4)
|
||||
ntype = argv[4][0];
|
||||
int n_classes = 3;
|
||||
if(argc > 5)
|
||||
n_classes = atoi(argv[5]);
|
||||
int n_batch = 1;
|
||||
if(argc > 6)
|
||||
n_batch = atoi(argv[6]);
|
||||
|
||||
bool show = true;
|
||||
if(argc > 7)
|
||||
show = atoi(argv[7]);
|
||||
float conf_thresh=0.3;
|
||||
if(argc > 8)
|
||||
conf_thresh = atof(argv[8]);
|
||||
|
||||
if(n_batch < 1 || n_batch > 64)
|
||||
FatalError("Batch dim not supported");
|
||||
|
||||
if(!show)
|
||||
SAVE_RESULT = true;
|
||||
|
||||
tk::dnn::CenternetDetection3D cnet;
|
||||
|
||||
tk::dnn::DetectionNN3D *detNN;
|
||||
|
||||
switch(ntype)
|
||||
{
|
||||
case 'c':
|
||||
detNN = &cnet;
|
||||
break;
|
||||
default:
|
||||
FatalError("Network type not allowed (3rd parameter)\n");
|
||||
}
|
||||
std::vector<cv::Mat> calibs;
|
||||
if(!calib_params.empty() && calib_params!="NULL") {
|
||||
std::cout<<"calib_params: "<<calib_params<<std::endl;
|
||||
cv::Mat calib;
|
||||
// the calibration matrix must be a 3x3 matrix
|
||||
readCalibrationMatrix(calib_params, calib);
|
||||
for(int bi=0; bi< n_batch; ++bi)
|
||||
calibs.push_back(calib);
|
||||
}
|
||||
detNN->init(net, n_classes, n_batch, conf_thresh, calibs);
|
||||
|
||||
gRun = true;
|
||||
|
||||
cv::VideoCapture cap(input);
|
||||
if(!cap.isOpened())
|
||||
gRun = false;
|
||||
else
|
||||
std::cout<<"camera started\n";
|
||||
|
||||
cv::VideoWriter resultVideo;
|
||||
if(SAVE_RESULT) {
|
||||
int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
|
||||
int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
|
||||
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
|
||||
}
|
||||
cv::Mat frame;
|
||||
if(show)
|
||||
cv::namedWindow("detection", cv::WINDOW_NORMAL);
|
||||
|
||||
std::vector<cv::Mat> batch_frame;
|
||||
std::vector<cv::Mat> batch_dnn_input;
|
||||
|
||||
while(gRun) {
|
||||
batch_dnn_input.clear();
|
||||
batch_frame.clear();
|
||||
|
||||
for(int bi=0; bi< n_batch; ++bi){
|
||||
cap >> frame;
|
||||
if(!frame.data)
|
||||
break;
|
||||
batch_frame.push_back(frame);
|
||||
|
||||
// this will be resized to the net format
|
||||
batch_dnn_input.push_back(frame.clone());
|
||||
}
|
||||
if(!frame.data)
|
||||
break;
|
||||
|
||||
//inference
|
||||
detNN->update(batch_dnn_input, n_batch, false, nullptr, false);
|
||||
detNN->draw(batch_frame);
|
||||
|
||||
if(show){
|
||||
for(int bi=0; bi< n_batch; ++bi){
|
||||
cv::imshow("detection", batch_frame[bi]);
|
||||
cv::waitKey(1);
|
||||
}
|
||||
}
|
||||
if(n_batch == 1 && SAVE_RESULT)
|
||||
resultVideo << frame;
|
||||
}
|
||||
|
||||
std::cout<<"detection end\n";
|
||||
double mean = 0;
|
||||
|
||||
std::cout<<COL_GREENB<<"\n\nTime preprocessing stats:\n";
|
||||
std::cout<<"Min: "<<*std::min_element(detNN->pre_stats.begin(), detNN->pre_stats.end())<<" ms\n";
|
||||
std::cout<<"Max: "<<*std::max_element(detNN->pre_stats.begin(), detNN->pre_stats.end())<<" ms\n";
|
||||
for(int i=0; i<detNN->pre_stats.size(); i++) mean += detNN->pre_stats[i]; mean /= detNN->pre_stats.size();
|
||||
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
|
||||
mean=0;
|
||||
std::cout<<COL_GREENB<<"\n\nTime stats:\n";
|
||||
std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
|
||||
std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
|
||||
for(int i=0; i<detNN->stats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size();
|
||||
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
|
||||
mean=0;
|
||||
std::cout<<COL_GREENB<<"\n\nTime postprocessing stats:\n";
|
||||
std::cout<<"Min: "<<*std::min_element(detNN->post_stats.begin(), detNN->post_stats.end())<<" ms\n";
|
||||
std::cout<<"Max: "<<*std::max_element(detNN->post_stats.begin(), detNN->post_stats.end())<<" ms\n";
|
||||
for(int i=0; i<detNN->post_stats.size(); i++) mean += detNN->post_stats[i]; mean /= detNN->post_stats.size();
|
||||
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
|
||||
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
#include <iostream>
|
||||
#include <signal.h>
|
||||
#include <stdlib.h> /* srand, rand */
|
||||
//#include <unistd.h>
|
||||
#include <mutex>
|
||||
|
||||
#include "demo_utils.h"
|
||||
#include "CenterTrack.h"
|
||||
|
||||
bool gRun;
|
||||
bool SAVE_RESULT = false;
|
||||
|
||||
void sig_handler(int signo) {
|
||||
std::cout<<"request gateway stop\n";
|
||||
gRun = false;
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
|
||||
std::cout<<"detection\n";
|
||||
signal(SIGINT, sig_handler);
|
||||
|
||||
|
||||
std::string net = "dla34_cnet3d_track_fp32.rt";
|
||||
if(argc > 1)
|
||||
net = argv[1];
|
||||
#ifdef __linux__
|
||||
std::string input = "../demo/yolo_test.mp4";
|
||||
#elif _WIN32
|
||||
std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
|
||||
#endif
|
||||
|
||||
if(argc > 2)
|
||||
input = argv[2];
|
||||
std::string calib_params = "";
|
||||
if(argc > 3)
|
||||
calib_params = argv[3];
|
||||
char ntype = 'c';
|
||||
if(argc > 4)
|
||||
ntype = argv[4][0];
|
||||
int n_classes = 3;
|
||||
if(argc > 5)
|
||||
n_classes = atoi(argv[5]);
|
||||
int n_batch = 1;
|
||||
if(argc > 6)
|
||||
n_batch = atoi(argv[6]);
|
||||
bool show = true;
|
||||
if(argc > 7)
|
||||
show = atoi(argv[7]);
|
||||
float conf_thresh=0.3;
|
||||
if(argc > 8)
|
||||
conf_thresh = atof(argv[8]);
|
||||
bool t3d = true;
|
||||
if(argc > 9)
|
||||
t3d = atoi(argv[9]);
|
||||
if(n_batch < 1 || n_batch > 64)
|
||||
FatalError("Batch dim not supported");
|
||||
|
||||
if(!show)
|
||||
SAVE_RESULT = true;
|
||||
|
||||
tk::dnn::CenterTrack ctrack;
|
||||
|
||||
tk::dnn::TrackingNN *trackNN;
|
||||
|
||||
switch(ntype)
|
||||
{
|
||||
case 'c':
|
||||
trackNN = &ctrack;
|
||||
break;
|
||||
default:
|
||||
FatalError("Network type not allowed (3rd parameter)\n");
|
||||
}
|
||||
std::vector<cv::Mat> calibs;
|
||||
if(!calib_params.empty() && calib_params!="NULL") {
|
||||
std::cout<<"calib_params: "<<calib_params<<std::endl;
|
||||
cv::Mat calib;
|
||||
// the calibration matrix must be a 3x3 matrix
|
||||
readCalibrationMatrix(calib_params, calib);
|
||||
for(int bi=0; bi< n_batch; ++bi)
|
||||
calibs.push_back(calib);
|
||||
}
|
||||
trackNN->init(net, n_classes, n_batch, conf_thresh, t3d, calibs);
|
||||
|
||||
gRun = true;
|
||||
|
||||
cv::VideoCapture cap(input);
|
||||
if(!cap.isOpened())
|
||||
gRun = false;
|
||||
else
|
||||
std::cout<<"camera started\n";
|
||||
|
||||
cv::VideoWriter resultVideo;
|
||||
if(SAVE_RESULT) {
|
||||
int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
|
||||
int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
|
||||
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
|
||||
}
|
||||
cv::Mat frame;
|
||||
if(show)
|
||||
cv::namedWindow("detection", cv::WINDOW_NORMAL);
|
||||
|
||||
std::vector<cv::Mat> batch_frame;
|
||||
std::vector<cv::Mat> batch_dnn_input;
|
||||
|
||||
while(gRun) {
|
||||
batch_dnn_input.clear();
|
||||
batch_frame.clear();
|
||||
|
||||
for(int bi=0; bi< n_batch; ++bi){
|
||||
cap >> frame;
|
||||
if(!frame.data)
|
||||
break;
|
||||
batch_frame.push_back(frame);
|
||||
|
||||
// this will be resized to the net format
|
||||
batch_dnn_input.push_back(frame.clone());
|
||||
}
|
||||
if(!frame.data)
|
||||
break;
|
||||
|
||||
//inference
|
||||
trackNN->update(batch_dnn_input, n_batch, false, nullptr, false);
|
||||
trackNN->draw(batch_frame);
|
||||
|
||||
if(show){
|
||||
for(int bi=0; bi< n_batch; ++bi){
|
||||
cv::imshow("detection", batch_frame[bi]);
|
||||
cv::waitKey(1);
|
||||
}
|
||||
}
|
||||
if(n_batch == 1 && SAVE_RESULT)
|
||||
resultVideo << frame;
|
||||
}
|
||||
|
||||
std::cout<<"detection end\n";
|
||||
double mean = 0;
|
||||
|
||||
std::cout<<COL_GREENB<<"\n\nTime preprocessing stats:\n";
|
||||
std::cout<<"Min: "<<*std::min_element(trackNN->pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n";
|
||||
std::cout<<"Max: "<<*std::max_element(trackNN->pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n";
|
||||
for(int i=0; i<trackNN->pre_stats.size(); i++) mean += trackNN->pre_stats[i]; mean /= trackNN->pre_stats.size();
|
||||
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
|
||||
mean=0;
|
||||
std::cout<<COL_GREENB<<"\n\nTime stats:\n";
|
||||
std::cout<<"Min: "<<*std::min_element(trackNN->stats.begin(), trackNN->stats.end())<<" ms\n";
|
||||
std::cout<<"Max: "<<*std::max_element(trackNN->stats.begin(), trackNN->stats.end())<<" ms\n";
|
||||
for(int i=0; i<trackNN->stats.size(); i++) mean += trackNN->stats[i]; mean /= trackNN->stats.size();
|
||||
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
|
||||
mean=0;
|
||||
std::cout<<COL_GREENB<<"\n\nTime postprocessing stats:\n";
|
||||
std::cout<<"Min: "<<*std::min_element(trackNN->post_stats.begin(), trackNN->post_stats.end())<<" ms\n";
|
||||
std::cout<<"Max: "<<*std::max_element(trackNN->post_stats.begin(), trackNN->post_stats.end())<<" ms\n";
|
||||
for(int i=0; i<trackNN->post_stats.size(); i++) mean += trackNN->post_stats[i]; mean /= trackNN->post_stats.size();
|
||||
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
|
||||
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# 2D/3D Object Detection and Tracking
|
||||
|
||||
Currently tkDNN supports only CenterTrack as 3DOD & 2D/3D Tracker network.
|
||||
|
||||
## 3D Object Detection
|
||||
|
||||
To run the 3D object detection demo follow these steps (example with CenterNet based on DLA34):
|
||||
```
|
||||
rm dla34_cnet3d_fp32.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_dla34_cnet3d # run the yolo test (is slow)
|
||||
./demo3D dla34_cnet3d_fp32.rt ../demo/yolo_test.mp4 NULL c
|
||||
```
|
||||
The demo3D program takes the same parameters of the demo program:
|
||||
```
|
||||
./demo3D <network-rt-file> <path-to-video> <calibration-file> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh>
|
||||
```
|
||||
where
|
||||
|
||||
* ```<calibration-file>``` is the camera calibration file (opencv format). It is important that the file contains entry "camera_matrix" with sub-entry "rows", "cols", "data". If you do not want to pass the calibration file, pass "NULL" instead.
|
||||
|
||||
|
||||
|
||||
## Object Detection and Tracking
|
||||
|
||||
To run the 3D object detection & tracking demo follow these steps (example with CenterTrack based on DLA34):
|
||||
```
|
||||
rm dla34_ctrack_fp32.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_dla34_ctrack # run the yolo test (is slow)
|
||||
./demoTracker dla34_ctrack_fp32.rt ../demo/yolo_test.mp4 NULL c
|
||||
```
|
||||
|
||||
The demoTracker program takes the same parameters of the demo program:
|
||||
```
|
||||
./demoTracker <network-rt-file> <path-to-video> <calibration-file> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh> <2D/3D-flag>
|
||||
```
|
||||
|
||||
where
|
||||
|
||||
* ```<calibration-file>``` is the camera calibration file (opencv format). It is important that the file contains entry "camera_matrix" with sub-entry "rows", "cols", "data". If you do not want to pass the calibration file, pass "NULL" instead.
|
||||
* ```<2D/3D-flag>``` if set to 0 the demo will be in the 2D mode, while if set to 1 the demo will be in the 3D mode (Default is 1 - 3D mode).
|
||||
|
||||
|
||||
|
||||
## FPS Results
|
||||
|
||||
Inference FPS of shelfnet with tkDNN, average of 1200 images on:
|
||||
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
|
||||
* Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
|
||||
|
||||
### 3D OD and Tracking
|
||||
|
||||
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
|
||||
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
|
||||
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | pre | 4.43883 | 225.285 | 4.42951 | 225.759 | 4.44278 | 225.084 |
|
||||
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | inf | 9.03454 | 110.686 | 6.02013 | 166.109 | 5.31611 | 188.108 |
|
||||
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | post | 0.96631 | 1034.87 | 0.96824 | 1032.80 | 0.95066 | 1051.90 |
|
||||
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | tot | 14.4397 | 69.2535 | 11.4179 | 87.5818 | 10.7095 | 93.3750 |
|
||||
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | pre | 4.60075 | 217.356 | 4.28658 | 233.286 | 4.29473 | 232.844 |
|
||||
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | inf | 8.48365 | 117.874 | 5.25150 | 190.422 | 4.58463 | 218.120 |
|
||||
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | post | 0.99484 | 1005.19 | 0.91776 | 1089.61 | 0.89853 | 1112.93 |
|
||||
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | tot | 14.0792 | 71.0266 | 10.4558 | 95.6405 | 9.77788 | 102.272 |
|
||||
| AGX Xavier | CenterTrack3D 512x512 (B=1) | pre | 34.9915 | 28.5784 | 33.5976 | 29.7440 | 34.4425 | 29.0339 |
|
||||
| AGX Xavier | CenterTrack3D 512x512 (B=1) | inf | 76.3579 | 13.0962 | 52.4759 | 19.0564 | 51.4610 | 19.4322 |
|
||||
| AGX Xavier | CenterTrack3D 512x512 (B=1) | post | 3.38576 | 295.355 | 3.26010 | 306.739 | 3.19770 | 312.725 |
|
||||
| AGX Xavier | CenterTrack3D 512x512 (B=1) | tot | 114.735 | 8.71574 | 89.3336 | 11.1940 | 89.1012 | 11.2232 |
|
||||
| AGX Xavier | CenterTrack3D 512x512 (B=4) | pre | 32.8933 | 30.4014 | 32.7950 | 30.4925 | 32.9603 | 30.3396 |
|
||||
| AGX Xavier | CenterTrack3D 512x512 (B=4) | inf | 74.2840 | 13.4618 | 50.3858 | 19.8469 | 49.2030 | 20.3240 |
|
||||
| AGX Xavier | CenterTrack3D 512x512 (B=4) | post | 3.14888 | 317.574 | 3.13615 | 318.862 | 3.02550 | 330.524 |
|
||||
| AGX Xavier | CenterTrack3D 512x512 (B=4) | tot | 110.326 | 9.06404 | 86.3169 | 11.5852 | 85.1888 | 11.7386 |
|
||||
|
||||
|
||||
### 2D OD and Tracking
|
||||
|
||||
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
|
||||
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
|
||||
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | pre | 4.44386 | 225.030 | 4.43828 | 225.313 | 4.47747 | 223.340 |
|
||||
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | inf | 9.08365 | 110.088 | 6.04842 | 165.332 | 5.34787 | 186.990 |
|
||||
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | post | 0.98593 | 1014.27 | 0.97745 | 1023.07 | 0.96595 | 1035.25 |
|
||||
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | tot | 14.5134 | 68.9018 | 11.4642 | 87.2281 | 10.7913 | 92.6672 |
|
||||
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | pre | 4.41188 | 226.661 | 4.50800 | 221.828 | 4.29238 | 232.971 |
|
||||
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | inf | 8.29015 | 120.625 | 5.38630 | 185.656 | 4.58500 | 218.103 |
|
||||
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | post | 0.96847 | 1032.55 | 0.97997 | 1020.44 | 0.91791 | 1089.43 |
|
||||
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | tot | 13.6705 | 73.1502 | 10.8743 | 91.9602 | 9.79528 | 102.090 |
|
||||
| AGX Xavier | CenterTrack2D 512x512 (B=1) | pre | 33.4745 | 29.8735 | 33.4847 | 29.8643 | 33.5022 | 29.8488 |
|
||||
| AGX Xavier | CenterTrack2D 512x512 (B=1) | inf | 76.2077 | 13.1220 | 52.5111 | 19.0436 | 51.6057 | 19.3777 |
|
||||
| AGX Xavier | CenterTrack2D 512x512 (B=1) | post | 3.26055 | 306.697 | 3.26806 | 305.992 | 3.21988 | 310.571 |
|
||||
| AGX Xavier | CenterTrack2D 512x512 (B=1) | tot | 111.943 | 8.93312 | 89.2639 | 11.2027 | 88.3278 | 11.3215 |
|
||||
| AGX Xavier | CenterTrack2D 512x512 (B=4) | pre | 32.8323 | 30.4579 | 32.8595 | 30.4326 | 32.8195 | 30.4697 |
|
||||
| AGX Xavier | CenterTrack2D 512x512 (B=4) | inf | 74.3075 | 13.4576 | 50.3555 | 19.8588 | 49.1805 | 20.3333 |
|
||||
| AGX Xavier | CenterTrack2D 512x512 (B=4) | post | 3.12360 | 320.143 | 3.13570 | 318.908 | 3.04943 | 327.931 |
|
||||
| AGX Xavier | CenterTrack2D 512x512 (B=4) | tot | 110.263 | 9.06920 | 86.3507 | 11.5807 | 85.0494 | 11.7579 |
|
||||
|
||||
+2
-23
@@ -2,19 +2,6 @@
|
||||
|
||||
Currently tkDNN supports only ShelfNet as semantic segmentation network.
|
||||
|
||||
## Export weights from Shelfnet
|
||||
To get the weights needed to run Shelfnet tests use [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of a Pytorch implementation of Shelfnet network.
|
||||
|
||||
```
|
||||
git clone https://git.hipert.unimore.it/mverucchi/shelfnet
|
||||
cd shelfnet
|
||||
cd ShelfNet18_realtime
|
||||
conda env create --file shelfnet_env.yml
|
||||
conda activate shelfnet
|
||||
mkdir layer debug
|
||||
python export.py
|
||||
```
|
||||
|
||||
|
||||
## Run the demo
|
||||
|
||||
@@ -42,19 +29,11 @@ where
|
||||
NB) By default it is used FP32 inference
|
||||
NB) The batching is not used to work on more streams, rather to work on more tiles of the same image. Shelfnet never resized the input image, therefore for images greater than 1024x1024 tiles of 1024x1024 are given in input to the network in batch.
|
||||
|
||||
|
||||
|
||||
|
||||
For other demo videos refer to [this playlist](https://www.youtube.com/playlist?list=PLv0nEQYDD45y5EdSiywwCGPBmJVUzIWwe).
|
||||
|
||||
|
||||
## Existing tests and supported networks
|
||||
|
||||
| Test Name | Network | Dataset | N Classes | Input size | Weights |
|
||||
| :---------------- | :-------------------------------------------- | :-----------------------------------------------------------: | :-------: | :-----------: | :------------------------------------------------------------------------ |
|
||||
| shelfnet | ShelfNet18_realtime<sup>1</sup> | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) |
|
||||
| shelfnet_berkeley | ShelfNet18_realtime<sup>1</sup> | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) |
|
||||
|
||||
1. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019.
|
||||
NB) The gif and the videos are obtained with Mapillary Vistas weights, that we cannot publicly share due to its license restrictions. However, you can train Shelfnet using Mapillary and [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of the original repo.
|
||||
|
||||
|
||||
## FPS Results
|
||||
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
# 2D Object Detection with tkDNN
|
||||
|
||||
## Supported Networks
|
||||
|
||||
* Yolo4, Yolo4-csp, Yolo4x, Yolo4_berkeley, Yolo4tiny
|
||||
* Yolo3, Yolo3_berkeley, Yolo3_coco4, Yolo3_flir, Yolo3_512, Yolo3tiny, Yolo3tiny_512
|
||||
* Yolo2, Yolo2_voc, Yolo2tiny
|
||||
* Csresnext50-panet-spp, Csresnext50-panet-spp_berkeley
|
||||
* Resnet101_cnet, Dla34_cnet
|
||||
* Mobilenetv2ssd, Mobilenetv2ssd512, Bdd-mobilenetv2ssd
|
||||
|
||||
## Index
|
||||
|
||||
- [2D Object Detection](#2d-object-detection)
|
||||
- [FP16 inference](#fp16-inference)
|
||||
- [INT8 inference](#int8-inference)
|
||||
- [Batching](#batching)
|
||||
|
||||
### 2D Object Detection
|
||||
This is an example using yolov4.
|
||||
|
||||
To run the an object detection first create the .rt file by running:
|
||||
```
|
||||
rm yolo4_fp32.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_yolo4 # run the yolo test (is slow)
|
||||
```
|
||||
If you get problems in the creation, try to check the error activating the debug of TensorRT in this way:
|
||||
```
|
||||
cmake .. -DDEBUG=True
|
||||
make
|
||||
```
|
||||
|
||||
Once you have successfully created your rt file, run the demo:
|
||||
```
|
||||
./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y
|
||||
```
|
||||
In general the demo program takes 7 parameters:
|
||||
```
|
||||
./demo <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh>
|
||||
```
|
||||
where
|
||||
|
||||
* ```<network-rt-file>``` is the rt file generated by a test
|
||||
* ```<<path-to-video>``` is the path to a video file or a camera input
|
||||
* ```<kind-of-network>``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
|
||||
* ```<number-of-classes>```is the number of classes the network is trained on
|
||||
* ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
|
||||
* ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
|
||||
* ```<conf-thresh>``` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed.
|
||||
|
||||
N.B. By default it is used FP32 inference
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### FP16 inference
|
||||
|
||||
To run the demo with FP16 inference follow these steps (example with yolov3):
|
||||
```
|
||||
export TKDNN_MODE=FP16 # set the half floating point optimization
|
||||
rm yolo3_fp16.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_yolo3 # run the yolo test (is slow)
|
||||
./demo yolo3_fp16.rt ../demo/yolo_test.mp4 y
|
||||
```
|
||||
N.B. Using FP16 inference will lead to some errors in the results (first or second decimal).
|
||||
|
||||
### INT8 inference
|
||||
|
||||
To run the demo with INT8 inference three environment variables need to be set:
|
||||
|
||||
* ```export TKDNN_MODE=INT8```: set the 8-bit integer optimization
|
||||
* ```export TKDNN_CALIB_IMG_PATH=/path/to/calibration/image_list.txt``` : image_list.txt has in each line the absolute path to a calibration image
|
||||
* ```export TKDNN_CALIB_LABEL_PATH=/path/to/calibration/label_list.txt```: label_list.txt has in each line the absolute path to a calibration label
|
||||
|
||||
You should provide image_list.txt and label_list.txt, using training images. However, if you want to quickly test the INT8 inference you can run (from this repo root folder)
|
||||
```
|
||||
bash scripts/download_validation.sh COCO
|
||||
```
|
||||
to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD instead of COCO to download BDD validation.
|
||||
|
||||
Then a complete example using yolo3 and COCO dataset would be:
|
||||
```
|
||||
export TKDNN_MODE=INT8
|
||||
export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
|
||||
export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
|
||||
rm yolo3_int8.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_yolo3 # run the yolo test (is slow)
|
||||
./demo yolo3_int8.rt ../demo/yolo_test.mp4 y
|
||||
```
|
||||
N.B.
|
||||
|
||||
* Using INT8 inference will lead to some errors in the results.
|
||||
* The test will be slower: this is due to the INT8 calibration, which may take some time to complete.
|
||||
* INT8 calibration requires TensorRT version greater than or equal to 6.0
|
||||
* Only 100 images are used to create the calibration table by default (set in the code).
|
||||
|
||||
### Batching
|
||||
|
||||
#### BatchSize bigger than 1
|
||||
```
|
||||
export TKDNN_BATCHSIZE=2
|
||||
# build tensorRT files
|
||||
```
|
||||
This will create a TensorRT file with the desired **max** batch size.
|
||||
The test will still run with a batch of 1, but the created tensorRT can manage the desired batch size.
|
||||
|
||||
#### Test batch Inference
|
||||
This will test the network with random input and check if the output of each batch is the same.
|
||||
```
|
||||
./test_rtinference <network-rt-file> <number-of-batches>
|
||||
# <number-of-batches> should be less or equal to the max batch size of the <network-rt-file>
|
||||
|
||||
# example
|
||||
export TKDNN_BATCHSIZE=4 # set max batch size
|
||||
rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files
|
||||
./test_yolo3 # build RT file
|
||||
./test_rtinference yolo3_fp32.rt 4 # test with a batch size of 4
|
||||
```
|
||||
@@ -0,0 +1,115 @@
|
||||
# tkDNN export weights
|
||||
|
||||
## Index
|
||||
|
||||
- [How to export weights](#how-to-export-weights)
|
||||
- [1)Export weights from darknet](#1export-weights-from-darknet)
|
||||
- [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101)
|
||||
- [3)Export weights for CenterNet](#3export-weights-for-centernet)
|
||||
- [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd)
|
||||
- [5)Export weights for CenterTrack](#5export-weights-for-centertrack)
|
||||
- [6)Export weights for ShelfNet](#6export-weights-for-shelfnet)
|
||||
- [Darknet Parser](#darknet-parser)
|
||||
|
||||
## How to export weights
|
||||
|
||||
Weights are essential for any network to run inference. For each test a folder organized as follow is needed (in the build folder):
|
||||
```
|
||||
test_nn
|
||||
|---- layers/ (folder containing a binary file for each layer with the corresponding wieghts and bias)
|
||||
|---- debug/ (folder containing a binary file for each layer with the corresponding outputs)
|
||||
```
|
||||
Therefore, once the weights have been exported, the folders layers and debug should be placed in the corresponding test.
|
||||
|
||||
### 1)Export weights from darknet
|
||||
To export weights for NNs that are defined in darknet framework, use [this](https://git.hipert.unimore.it/fgatti/darknet.git) fork of darknet and follow these steps to obtain a correct debug and layers folder, ready for tkDNN.
|
||||
|
||||
```
|
||||
git clone https://git.hipert.unimore.it/fgatti/darknet.git
|
||||
cd darknet
|
||||
make
|
||||
mkdir layers debug
|
||||
./darknet export <path-to-cfg-file> <path-to-weights> layers
|
||||
```
|
||||
N.B. Use compilation with CPU (leave GPU=0 in Makefile) if you also want debug.
|
||||
|
||||
### 2)Export weights for DLA34 and ResNet101
|
||||
To get weights and outputs needed to run the tests dla34 and resnet101 use the Python script and the Anaconda environment included in the repository.
|
||||
|
||||
Create Anaconda environment and activate it:
|
||||
```
|
||||
conda env create -f file_name.yml
|
||||
source activate env_name
|
||||
python <script name>
|
||||
```
|
||||
### 3)Export weights for CenterNet
|
||||
To get the weights needed to run Centernet tests use [this](https://github.com/sapienzadavide/CenterNet.git) fork of the original Centernet.
|
||||
```
|
||||
git clone https://github.com/sapienzadavide/CenterNet.git
|
||||
```
|
||||
* follow the instruction in the README.md and INSTALL.md
|
||||
|
||||
```
|
||||
python demo.py --input_res 512 --arch resdcn_101 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_resdcn101.pth --exp_wo --exp_wo_dim 512
|
||||
python demo.py --input_res 512 --arch dla_34 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_dla_2x.pth --exp_wo --exp_wo_dim 512
|
||||
```
|
||||
### 4)Export weights for MobileNetSSD
|
||||
To get the weights needed to run Mobilenet tests use [this](https://github.com/mive93/pytorch-ssd) fork of a Pytorch implementation of SSD network.
|
||||
|
||||
```
|
||||
git clone https://github.com/mive93/pytorch-ssd
|
||||
cd pytorch-ssd
|
||||
conda env create -f env_mobv2ssd.yml
|
||||
python run_ssd_live_demo.py mb2-ssd-lite <pth-model-fil> <labels-file>
|
||||
```
|
||||
### 5)Export weights for CenterTrack
|
||||
To get the weights needed to run CenterTrack tests use [this](https://github.com/sapienzadavide/CenterTrack.git) fork of the original CenterTrack.
|
||||
```
|
||||
git clone https://github.com/sapienzadavide/CenterTrack.git
|
||||
```
|
||||
* follow the instruction in the README.md and INSTALL.md
|
||||
|
||||
```
|
||||
python demo.py tracking,ddd --load_model ../models/nuScenes_3Dtracking.pth --dataset nuscenes --pre_hm --track_thresh 0.1 --demo /path/to/image/or/folder/or/video/or/webcam --test_focal_length 633 --exp_wo --exp_wo_dim 512 --input_h 512 --input_w 512
|
||||
```
|
||||
|
||||
### 6)Export weights for ShelfNet
|
||||
To get the weights needed to run Shelfnet tests use [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of a Pytorch implementation of Shelfnet network.
|
||||
|
||||
```
|
||||
git clone https://git.hipert.unimore.it/mverucchi/shelfnet
|
||||
cd shelfnet
|
||||
cd ShelfNet18_realtime
|
||||
conda env create --file shelfnet_env.yml
|
||||
conda activate shelfnet
|
||||
mkdir layer debug
|
||||
python export.py
|
||||
```
|
||||
|
||||
## Darknet Parser
|
||||
tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*:
|
||||
```
|
||||
// example of parsing yolo4
|
||||
tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names");
|
||||
net->print();
|
||||
```
|
||||
All models from darknet are now parsed directly from cfg, you still need to export the weights with the described tools in the previous section.
|
||||
<details>
|
||||
<summary>Supported layers</summary>
|
||||
convolutional
|
||||
maxpool
|
||||
avgpool
|
||||
shortcut
|
||||
upsample
|
||||
route
|
||||
reorg
|
||||
region
|
||||
yolo
|
||||
</details>
|
||||
<details>
|
||||
<summary>Supported activations</summary>
|
||||
relu
|
||||
leaky
|
||||
mish
|
||||
logistic
|
||||
</details>
|
||||
@@ -0,0 +1,32 @@
|
||||
# Run the mAP demo
|
||||
|
||||
To compute mAP, precision, recall and f1score to evaluate 2D object detectors, run the map_demo.
|
||||
|
||||
A validation set is needed.
|
||||
To download COCO_val2017 (80 classes) run (form the root folder):
|
||||
```
|
||||
bash scripts/download_validation.sh COCO
|
||||
```
|
||||
To download Berkeley_val (10 classes) run (form the root folder):
|
||||
```
|
||||
bash scripts/download_validation.sh BDD
|
||||
```
|
||||
|
||||
To compute the map, the following parameters are needed:
|
||||
```
|
||||
./map_demo <network rt> <network type [y|c|m]> <labels file path> <config file path>
|
||||
```
|
||||
where
|
||||
* ```<network rt>```: rt file of a chosen network on which compute the mAP.
|
||||
* ```<network type [y|c|m]>```: type of network. Right now only y(yolo), c(centernet) and m(mobilenet) are allowed
|
||||
* ```<labels file path>```: path to a text file containing all the paths of the ground-truth labels. It is important that all the labels of the ground-truth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the ground-truth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg.
|
||||
* ```<config file path>```: path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
cd build
|
||||
./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml
|
||||
```
|
||||
|
||||
This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to submit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate).
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 6.3 MiB |
@@ -0,0 +1,95 @@
|
||||
# tkDNN on Windows
|
||||
|
||||
## Index
|
||||
|
||||
- [Dependencies-Windows](#dependencies-windows)
|
||||
- [Compiling tkDNN on Windows](#compiling-tkdnn-on-windows)
|
||||
- [Run the demo on Windows](#run-the-demo-on-windows)
|
||||
- [FP16 inference windows](#fp16-inference-windows)
|
||||
- [INT8 inference windows](#int8-inference-windows)
|
||||
- [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows)
|
||||
|
||||
### Dependencies-Windows
|
||||
This branch should work on every NVIDIA GPU supported in windows with the following dependencies:
|
||||
|
||||
* WINDOWS 10 1803 or HIGHER
|
||||
* CUDA 10.0 (Recommended CUDA 11.2 )
|
||||
* CUDNN 7.6 (Recommended CUDNN 8.1.1 )
|
||||
* TENSORRT 6.0.1 (Recommended TENSORRT 7.2.3.4 )
|
||||
* OPENCV 3.4 (Recommended OPENCV 4.2.0 )
|
||||
* MSVC 16.7
|
||||
* YAML-CPP
|
||||
* EIGEN3
|
||||
* 7ZIP (ADD TO PATH)
|
||||
* NINJA 1.10
|
||||
|
||||
|
||||
All the above mentioned dependencies except 7ZIP can be installed using Microsoft's [VCPKG](https://github.com/microsoft/vcpkg.git) .
|
||||
After bootstrapping VCPKG the dependencies can be built and installed using the following command :
|
||||
|
||||
```
|
||||
opencv4(normal) - vcpkg.exe install opencv4[tbb,jpeg,tiff,opengl,openmp,png,ffmpeg,eigen]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build
|
||||
|
||||
opencv4(cuda) - vcpkg.exe install opencv4[cuda,nonfree,contrib,eigen,tbb,jpeg,tiff,opengl,openmp,png,ffmpeg]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build
|
||||
```
|
||||
To build opencv4 with cuda and cudnn version corresponding to your cuda version,vcpkg's cudnn portfile needs to be modified by adding ```$ENV{CUDA_PATH}``` at lines 16 and 17 in the portfile.cmake
|
||||
|
||||
After VCPKG finishes building and installing all the packages delete C:\temp_vcpkg_build and add C:\opt\x64-windows\bin and C:\opt\x64-windows\debug\bin to path
|
||||
|
||||
### Compiling tkDNN on Windows
|
||||
|
||||
tkDNN is built with cmake(3.15+) on windows along with ninja.Msbuild and NMake Makefiles are drastically slower when compiling the library compared to windows
|
||||
```
|
||||
git clone https://github.com/ceccocats/tkDNN.git
|
||||
cd tkdnn-windows
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release -G"Ninja" ..
|
||||
ninja -j4
|
||||
```
|
||||
|
||||
### Run the demo on Windows
|
||||
|
||||
This example uses yolo4_tiny.\
|
||||
To run the object detection file create .rt file bu running:
|
||||
```
|
||||
.\test_yolo4tiny.exe
|
||||
```
|
||||
|
||||
Once the rt file has been successfully create,run the demo using the following command:
|
||||
```
|
||||
.\demo.exe yolo4tiny_fp32.rt ..\demo\yolo_test.mp4 y
|
||||
```
|
||||
For general info on more demo paramters,check Run the demo section on top
|
||||
To run the test_all_tests.sh on windows,use git bash or msys2
|
||||
|
||||
### FP16 inference windows
|
||||
|
||||
This is an untested feature on windows.To run the object detection demo with FP16 interference follow the below steps(example with yolo4tiny):
|
||||
```
|
||||
set TKDNN_MODE=FP16
|
||||
del /f yolo4tiny_fp16.rt
|
||||
.\test_yolo4tiny.exe
|
||||
.\demo.exe yolo4tiny_fp16.rt ..\demo\yolo_test.mp4
|
||||
```
|
||||
|
||||
### INT8 inference windows
|
||||
To run object detection demo with INT8 (example with yolo4tiny):
|
||||
```
|
||||
set TKDNN_MODE=INT8
|
||||
set TKDNN_CALIB_LABEL_PATH=..\demo\COCO_val2017\all_labels.txt
|
||||
set TKDNN_CALIB_IMG_PATH=..\demo\COCO_val2017\all_images.txt
|
||||
del /f yolo4tiny_int8.rt # be sure to delete(or move) old tensorRT files
|
||||
.\test_yolo4tiny.exe # run the yolo test (is slow)
|
||||
.\demo.exe yolo4tiny_int8.rt ..\demo\yolo_test.mp4 y
|
||||
|
||||
```
|
||||
|
||||
### Known issues with tkDNN on Windows
|
||||
|
||||
Mobilenet and Centernet demos work properly only when built with msvc 16.7 in Release Mode,when built in debug mode for the mentioned networks one might encounter opencv assert errors
|
||||
|
||||
All Darknet models work properly with demo using MSVC version(16.7-16.9)
|
||||
|
||||
It is recommended to use Nvidia Driver(465+),Cuda unknown errors have been observed when using older drivers on pascal(SM 61) devices.
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
#ifndef CENTERTRACK_H
|
||||
#define CENTERTRACK_H
|
||||
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include "opencv2/opencv.hpp"
|
||||
#include "kernels.h"
|
||||
#include "utils.h"
|
||||
#include "tkdnn.h"
|
||||
#include <time.h>
|
||||
#include <vector>
|
||||
#include <numeric> // std::iota
|
||||
#include <algorithm> // std::sort
|
||||
|
||||
#include "TrackingNN.h"
|
||||
|
||||
#include "kernelsThrust.h"
|
||||
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
struct detectionRes
|
||||
{
|
||||
float score;
|
||||
int cl;
|
||||
cv::Mat ct, tr, bb0, bb1;
|
||||
float dep;
|
||||
float dim[3];
|
||||
float alpha;
|
||||
float x,y,z;
|
||||
float rot_y;
|
||||
detectionRes() : ct(cv::Mat(cv::Size(1,2), CV_32F)),
|
||||
tr(cv::Mat(cv::Size(1,2), CV_32F)),
|
||||
bb0(cv::Mat(cv::Size(1,2), CV_32F)),
|
||||
bb1(cv::Mat(cv::Size(1,2), CV_32F)) { }
|
||||
~detectionRes() {
|
||||
ct.release();
|
||||
tr.release();
|
||||
bb0.release();
|
||||
bb1.release();
|
||||
}
|
||||
};
|
||||
|
||||
struct trackingRes
|
||||
{
|
||||
struct detectionRes det_res;
|
||||
int tracking_id;
|
||||
int age;
|
||||
int active;
|
||||
int color;
|
||||
};
|
||||
|
||||
class CenterTrack : public TrackingNN
|
||||
{
|
||||
public:
|
||||
tk::dnn::dataDim_t dim;
|
||||
tk::dnn::dataDim_t dim2;
|
||||
tk::dnn::dataDim_t dim_hm;
|
||||
tk::dnn::dataDim_t dim_wh;
|
||||
tk::dnn::dataDim_t dim_reg;
|
||||
tk::dnn::dataDim_t dim_track;
|
||||
tk::dnn::dataDim_t dim_dep;
|
||||
tk::dnn::dataDim_t dim_rot;
|
||||
tk::dnn::dataDim_t dim_dim;
|
||||
tk::dnn::dataDim_t dim_amodel_offset;
|
||||
|
||||
/* preprocessing */
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
float *mean_d;
|
||||
float *stddev_d;
|
||||
#else
|
||||
cv::Vec<float, 3> mean;
|
||||
cv::Vec<float, 3> stddev;
|
||||
dnnType *input;
|
||||
#endif
|
||||
float *d_ptrs;
|
||||
|
||||
std::vector<cv::Mat> inputCalibs;
|
||||
|
||||
std::vector<cv::Size> szOld;
|
||||
|
||||
cv::Mat src;
|
||||
cv::Mat dst;
|
||||
cv::Mat dst2;
|
||||
cv::Mat trans, trans2, transOut;
|
||||
|
||||
/* pre inf */
|
||||
bool iter0;
|
||||
dnnType *input_pre_inf_d;
|
||||
bool test_pre_inf = true;
|
||||
dnnType *img_d, *hm_d;
|
||||
tk::dnn::dataDim_t dim_in0;
|
||||
tk::dnn::dataDim_t dim_in1;
|
||||
dnnType *out_d;
|
||||
|
||||
|
||||
/* postprocessing */
|
||||
int K = 100;
|
||||
int width = 128;//56; // TODO
|
||||
|
||||
// pointer used in the kernels
|
||||
float *src_out;
|
||||
int *ids_out;
|
||||
|
||||
float *topk_scores;
|
||||
int *topk_inds_;
|
||||
float *topk_ys_;
|
||||
float *topk_xs_;
|
||||
int *ids_d, *ids_;
|
||||
|
||||
float *ones;
|
||||
|
||||
float *scores, *scores_d;
|
||||
int *clses, *clses_d;
|
||||
int *topk_inds_d;
|
||||
float *topk_ys_d;
|
||||
float *topk_xs_d;
|
||||
int *inttopk_xs_d, *inttopk_ys_d;
|
||||
|
||||
float *bbx0, *bby0, *bbx1, *bby1;
|
||||
float *bbx0_d, *bby0_d, *bbx1_d, *bby1_d;
|
||||
|
||||
int *intxs, *intys;
|
||||
|
||||
float *track, *dep, *rot, *dim_, *wh, *amodel_offset;
|
||||
float *track_d, *dep_d, *rot_d, *dim_d, *wh_d, *amodel_offset_d;
|
||||
|
||||
float *target_coords;
|
||||
|
||||
/* visualization */
|
||||
cv::Mat r;
|
||||
std::vector<cv::Mat> calibs;
|
||||
cv::Mat corners, pts3DHomo;
|
||||
|
||||
std::vector<std::vector<int>> faceId;
|
||||
cv::Scalar trColors[256];
|
||||
bool mode3D;
|
||||
|
||||
//processing
|
||||
struct threshold op;
|
||||
float outThresh = 0.1;
|
||||
float newThresh = 0.3;
|
||||
// float peakThreshold = 0.2;
|
||||
// float centerThreshold = 0.3; //default 0.5
|
||||
|
||||
|
||||
//detections
|
||||
std::vector<struct detectionRes> detRes;
|
||||
int countDet;
|
||||
//tracks
|
||||
std::vector<std::vector<struct trackingRes>> trRes;
|
||||
std::vector<int> countTr;
|
||||
std::vector<int> trackId;
|
||||
|
||||
|
||||
bool init_preprocessing();
|
||||
bool init_pre_inf();
|
||||
bool init_postprocessing();
|
||||
bool init_visualization(const int n_classes);
|
||||
void pre_inf(const int bi);
|
||||
void _get_additional_inputs();
|
||||
cv::Mat transform_preds_with_trans(float x1, float x2);
|
||||
void tracking(const int bi);
|
||||
|
||||
public:
|
||||
tk::dnn::Network *pre_phase_net = nullptr;
|
||||
CenterTrack() {};
|
||||
~CenterTrack() {};
|
||||
bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1,
|
||||
const float conf_thresh=0.3, const bool mode_3d=true,
|
||||
const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>());
|
||||
void preprocess(cv::Mat &frame, const int bi=0);
|
||||
void postprocess(const int bi=0,const bool mAP=false);
|
||||
void draw(std::vector<cv::Mat>& frames);
|
||||
};
|
||||
|
||||
|
||||
} // namespace dnn
|
||||
} // namespace tk
|
||||
|
||||
|
||||
#endif /*CENTERTRACK_H*/
|
||||
@@ -0,0 +1,101 @@
|
||||
#ifndef CENTERNETDETECTION3D_H
|
||||
#define CENTERNETDETECTION3D_H
|
||||
|
||||
#include "kernels.h"
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include "opencv2/opencv.hpp"
|
||||
#include <time.h>
|
||||
#include <vector>
|
||||
#include <numeric> // std::iota
|
||||
#include <algorithm> // std::sort
|
||||
|
||||
#include "DetectionNN3D.h"
|
||||
|
||||
#include "kernelsThrust.h"
|
||||
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
class CenternetDetection3D : public DetectionNN3D
|
||||
{
|
||||
private:
|
||||
tk::dnn::dataDim_t dim;
|
||||
tk::dnn::dataDim_t dim2;
|
||||
tk::dnn::dataDim_t dim_hm;
|
||||
tk::dnn::dataDim_t dim_wh;
|
||||
tk::dnn::dataDim_t dim_reg;
|
||||
tk::dnn::dataDim_t dim_dep;
|
||||
tk::dnn::dataDim_t dim_rot;
|
||||
tk::dnn::dataDim_t dim_dim;
|
||||
|
||||
std::vector<cv::Mat> inputCalibs;
|
||||
float *topk_scores;
|
||||
int *topk_inds_;
|
||||
float *topk_ys_;
|
||||
float *topk_xs_;
|
||||
int *ids_d, *ids_;
|
||||
|
||||
float *ones;
|
||||
|
||||
float *scores, *scores_d;
|
||||
int *clses, *clses_d;
|
||||
int *topk_inds_d;
|
||||
float *topk_ys_d;
|
||||
float *topk_xs_d;
|
||||
int *inttopk_xs_d, *inttopk_ys_d;
|
||||
|
||||
float *xs, *ys;
|
||||
|
||||
float *dep, *rot, *dim_, *wh;
|
||||
float *dep_d, *rot_d, *dim_d, *wh_d;
|
||||
|
||||
float *target_coords;
|
||||
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
float *mean_d;
|
||||
float *stddev_d;
|
||||
#else
|
||||
cv::Vec<float, 3> mean;
|
||||
cv::Vec<float, 3> stddev;
|
||||
dnnType *input;
|
||||
#endif
|
||||
cv::Mat r;
|
||||
float *d_ptrs;
|
||||
|
||||
cv::Size sz_old;
|
||||
|
||||
cv::Mat src;
|
||||
cv::Mat dst;
|
||||
cv::Mat dst2;
|
||||
cv::Mat trans, trans2;
|
||||
std::vector<cv::Mat> calibs;
|
||||
|
||||
//processing
|
||||
int K = 100;
|
||||
int width = 128;//56; // TODO
|
||||
|
||||
// pointer used in the kernels
|
||||
float *srcOut;
|
||||
int *idsOut;
|
||||
|
||||
struct threshold op;
|
||||
cv::Mat corners, pts3DHomo;
|
||||
|
||||
std::vector<std::vector<int>> faceId;
|
||||
|
||||
public:
|
||||
CenternetDetection3D() {};
|
||||
~CenternetDetection3D() {};
|
||||
|
||||
bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>());
|
||||
void preprocess(cv::Mat &frame, const int bi=0);
|
||||
void postprocess(const int bi=0,const bool mAP=false);
|
||||
void draw(std::vector<cv::Mat>& frames);
|
||||
};
|
||||
|
||||
|
||||
} // namespace dnn
|
||||
} // namespace tk
|
||||
|
||||
|
||||
#endif /*CENTERNETDETECTION_H*/
|
||||
@@ -0,0 +1,161 @@
|
||||
#ifndef DETECTIONNN3D_H
|
||||
#define DETECTIONNN3D_H
|
||||
|
||||
#include <iostream>
|
||||
#include <signal.h>
|
||||
#include <stdlib.h>
|
||||
#ifdef __linux__
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include <mutex>
|
||||
#include "utils.h"
|
||||
|
||||
#include <opencv2/core/core.hpp>
|
||||
#include <opencv2/highgui/highgui.hpp>
|
||||
#include <opencv2/imgproc/imgproc.hpp>
|
||||
|
||||
#include "tkdnn.h"
|
||||
|
||||
// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
|
||||
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
#include <opencv2/cudawarping.hpp>
|
||||
#include <opencv2/cudaarithm.hpp>
|
||||
#endif
|
||||
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
class DetectionNN3D {
|
||||
|
||||
protected:
|
||||
tk::dnn::NetworkRT *netRT = nullptr;
|
||||
dnnType *input_d;
|
||||
|
||||
std::vector<cv::Size> originalSize;
|
||||
|
||||
cv::Scalar colors[256];
|
||||
|
||||
int nBatches = 1;
|
||||
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
cv::cuda::GpuMat bgr[3];
|
||||
cv::cuda::GpuMat imagePreproc;
|
||||
#else
|
||||
cv::Mat bgr[3];
|
||||
cv::Mat imagePreproc;
|
||||
dnnType *input;
|
||||
#endif
|
||||
|
||||
/**
|
||||
* This method preprocess the image, before feeding it to the NN.
|
||||
*
|
||||
* @param frame original frame to adapt for inference.
|
||||
* @param bi batch index
|
||||
*/
|
||||
virtual void preprocess(cv::Mat &frame, const int bi=0) = 0;
|
||||
|
||||
/**
|
||||
* This method postprocess the output of the NN to obtain the correct
|
||||
* boundig boxes.
|
||||
*
|
||||
* @param bi batch index
|
||||
* @param mAP set to true only if all the probabilities for a bounding
|
||||
* box are needed, as in some cases for the mAP calculation
|
||||
*/
|
||||
virtual void postprocess(const int bi=0,const bool mAP=false) = 0;
|
||||
|
||||
public:
|
||||
int classes = 0;
|
||||
float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
|
||||
|
||||
std::vector<tk::dnn::box3D> detected3D; /*bounding boxes in output*/
|
||||
std::vector<std::vector<tk::dnn::box3D>> batchDetected; /*bounding boxes in output*/
|
||||
std::vector<double> pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/
|
||||
std::vector<std::string> classesNames;
|
||||
|
||||
DetectionNN3D() {};
|
||||
~DetectionNN3D(){};
|
||||
|
||||
/**
|
||||
* Method used to initialize the class, allocate memory and compute
|
||||
* needed data.
|
||||
*
|
||||
* @param tensor_path path to the rt file of the NN.
|
||||
* @param n_classes number of classes for the given dataset.
|
||||
* @param n_batches maximum number of batches to use in inference.
|
||||
* @return true if everything is correct, false otherwise.
|
||||
*/
|
||||
virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1,
|
||||
const float conf_thresh=0.3, const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>()) = 0;
|
||||
|
||||
/**
|
||||
* This method performs the whole detection of the NN.
|
||||
*
|
||||
* @param frames frames to run detection on.
|
||||
* @param cur_batches number of batches to use in inference.
|
||||
* @param save_times if set to true, preprocess, inference and postprocess times
|
||||
* are saved on a csv file, otherwise not.
|
||||
* @param times pointer to the output stream where to write times.
|
||||
* @param mAP set to true only if all the probabilities for a bounding
|
||||
* box are needed, as in some cases for the mAP calculation.
|
||||
*/
|
||||
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool save_times=false,
|
||||
std::ofstream *times=nullptr, const bool mAP=false){
|
||||
if(save_times && times==nullptr)
|
||||
FatalError("save_times set to true, but no valid ofstream given");
|
||||
if(cur_batches > nBatches)
|
||||
FatalError("A batch size greater than nBatches cannot be used");
|
||||
|
||||
originalSize.clear();
|
||||
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
|
||||
{
|
||||
TKDNN_TSTART
|
||||
for(int bi=0; bi<cur_batches;++bi){
|
||||
if(!frames[bi].data)
|
||||
FatalError("No image data feed to detection");
|
||||
originalSize.push_back(frames[bi].size());
|
||||
preprocess(frames[bi], bi);
|
||||
}
|
||||
TKDNN_TSTOP
|
||||
pre_stats.push_back(t_ns);
|
||||
if(save_times) *times<<t_ns<<";";
|
||||
}
|
||||
|
||||
//do inference
|
||||
tk::dnn::dataDim_t dim = netRT->input_dim;
|
||||
dim.n = cur_batches;
|
||||
{
|
||||
if(TKDNN_VERBOSE) dim.print();
|
||||
TKDNN_TSTART
|
||||
netRT->infer(dim, input_d);
|
||||
TKDNN_TSTOP
|
||||
if(TKDNN_VERBOSE) dim.print();
|
||||
stats.push_back(t_ns);
|
||||
if(save_times) *times<<t_ns<<";";
|
||||
}
|
||||
|
||||
batchDetected.clear();
|
||||
{
|
||||
TKDNN_TSTART
|
||||
for(int bi=0; bi<cur_batches;++bi)
|
||||
postprocess(bi, mAP);
|
||||
TKDNN_TSTOP
|
||||
post_stats.push_back(t_ns);
|
||||
if(save_times) *times<<t_ns<<"\n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to draw bounding boxes and labels on a frame.
|
||||
*
|
||||
* @param frames original frame to draw bounding box on.
|
||||
*/
|
||||
virtual void draw(std::vector<cv::Mat>& frames){};
|
||||
|
||||
};
|
||||
|
||||
}}
|
||||
|
||||
#endif /* DETECTIONNN3D_H*/
|
||||
@@ -620,6 +620,16 @@ struct sortable_bbox {
|
||||
int cl;
|
||||
float **probs;
|
||||
};
|
||||
struct box3D {
|
||||
int cl;
|
||||
std::vector<float> corners;
|
||||
float prob;
|
||||
|
||||
void print()
|
||||
{
|
||||
std::cout<<"\tcl: "<<cl<<"\tprob: "<<prob<<"\tshape corners: "<<corners.size()<<std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
Yolo3 layer
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
#ifndef TRACKINGNN_H
|
||||
#define TRACKINGNN_H
|
||||
|
||||
#include <iostream>
|
||||
#include <signal.h>
|
||||
#include <stdlib.h>
|
||||
#ifdef __linux__
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include <mutex>
|
||||
#include "utils.h"
|
||||
|
||||
#include <opencv2/core/core.hpp>
|
||||
#include <opencv2/highgui/highgui.hpp>
|
||||
#include <opencv2/imgproc/imgproc.hpp>
|
||||
|
||||
#include "tkdnn.h"
|
||||
|
||||
// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
|
||||
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
#include <opencv2/cudawarping.hpp>
|
||||
#include <opencv2/cudaarithm.hpp>
|
||||
#endif
|
||||
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
class TrackingNN {
|
||||
|
||||
protected:
|
||||
tk::dnn::NetworkRT *netRT = nullptr;
|
||||
dnnType *input_d;
|
||||
|
||||
std::vector<cv::Size> originalSize;
|
||||
|
||||
cv::Scalar colors[256];
|
||||
|
||||
int nBatches = 1;
|
||||
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
cv::cuda::GpuMat bgr[3];
|
||||
cv::cuda::GpuMat imagePreproc;
|
||||
#else
|
||||
cv::Mat bgr[3];
|
||||
cv::Mat imagePreproc;
|
||||
dnnType *input;
|
||||
#endif
|
||||
|
||||
/**
|
||||
* This method preprocess the image, before feeding it to the NN.
|
||||
*
|
||||
* @param frame original frame to adapt for inference.
|
||||
* @param bi batch index
|
||||
*/
|
||||
virtual void preprocess(cv::Mat &frame, const int bi=0) = 0;
|
||||
|
||||
/**
|
||||
* This method postprocess the output of the NN to obtain the correct
|
||||
* boundig boxes.
|
||||
*
|
||||
* @param bi batch index
|
||||
* @param mAP set to true only if all the probabilities for a bounding
|
||||
* box are needed, as in some cases for the mAP calculation
|
||||
*/
|
||||
virtual void postprocess(const int bi=0,const bool mAP=false) = 0;
|
||||
|
||||
public:
|
||||
int classes = 0;
|
||||
float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
|
||||
|
||||
std::vector<double> pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/
|
||||
std::vector<std::string> classesNames;
|
||||
|
||||
TrackingNN() {};
|
||||
~TrackingNN(){};
|
||||
|
||||
/**
|
||||
* Method used to initialize the class, allocate memory and compute
|
||||
* needed data.
|
||||
*
|
||||
* @param tensor_path path to the rt file of the NN.
|
||||
* @param n_classes number of classes for the given dataset.
|
||||
* @param n_batches maximum number of batches to use in inference.
|
||||
* @return true if everything is correct, false otherwise.
|
||||
*/
|
||||
virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1,
|
||||
const float conf_thresh=0.3, const bool mode_3d=true, const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>()) = 0;
|
||||
|
||||
/**
|
||||
* This method performs the whole detection and tracking of the NN.
|
||||
*
|
||||
* @param frames frames to run detection and trcking on.
|
||||
* @param cur_batches number of batches to use in inference.
|
||||
* @param save_times if set to true, preprocess, inference and postprocess times
|
||||
* are saved on a csv file, otherwise not.
|
||||
* @param times pointer to the output stream where to write times.
|
||||
* @param mAP set to true only if all the probabilities for a bounding
|
||||
* box are needed, as in some cases for the mAP calculation.
|
||||
*/
|
||||
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool save_times=false,
|
||||
std::ofstream *times=nullptr, const bool mAP=false){
|
||||
if(save_times && times==nullptr)
|
||||
FatalError("save_times set to true, but no valid ofstream given");
|
||||
if(cur_batches > nBatches)
|
||||
FatalError("A batch size greater than nBatches cannot be used");
|
||||
|
||||
originalSize.clear();
|
||||
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
|
||||
{
|
||||
TKDNN_TSTART
|
||||
for(int bi=0; bi<cur_batches;++bi){
|
||||
if(!frames[bi].data)
|
||||
FatalError("No image data feed to detection");
|
||||
originalSize.push_back(frames[bi].size());
|
||||
preprocess(frames[bi], bi);
|
||||
}
|
||||
TKDNN_TSTOP
|
||||
pre_stats.push_back(t_ns);
|
||||
if(save_times) *times<<t_ns<<";";
|
||||
}
|
||||
|
||||
//do inference
|
||||
tk::dnn::dataDim_t dim = netRT->input_dim;
|
||||
dim.n = cur_batches;
|
||||
{
|
||||
if(TKDNN_VERBOSE) dim.print();
|
||||
TKDNN_TSTART
|
||||
netRT->infer(dim, input_d);
|
||||
TKDNN_TSTOP
|
||||
if(TKDNN_VERBOSE) dim.print();
|
||||
stats.push_back(t_ns);
|
||||
if(save_times) *times<<t_ns<<";";
|
||||
}
|
||||
|
||||
{
|
||||
TKDNN_TSTART
|
||||
for(int bi=0; bi<cur_batches;++bi)
|
||||
postprocess(bi, mAP);
|
||||
TKDNN_TSTOP
|
||||
post_stats.push_back(t_ns);
|
||||
if(save_times) *times<<t_ns<<"\n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to draw bounding boxes and labels on a frame.
|
||||
*
|
||||
* @param frames original frame to draw bounding box on.
|
||||
*/
|
||||
virtual void draw(std::vector<cv::Mat>& frames){};
|
||||
|
||||
};
|
||||
|
||||
}}
|
||||
|
||||
#endif /* TRACKINGNN_H*/
|
||||
@@ -0,0 +1,22 @@
|
||||
#ifndef DEMO_UTILS_H
|
||||
#define DEMO_UTILS_H
|
||||
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <fstream>
|
||||
#include <iomanip>
|
||||
#include <stdlib.h>
|
||||
|
||||
#ifdef __linux__
|
||||
#include <unistd.h>
|
||||
#include <opencv2/core/core.hpp>
|
||||
#include <opencv2/highgui/highgui.hpp>
|
||||
#include <opencv2/imgproc/imgproc.hpp>
|
||||
|
||||
#include <yaml-cpp/yaml.h>
|
||||
#endif
|
||||
|
||||
|
||||
void readCalibrationMatrix(const std::string& path, cv::Mat& calib_mat);
|
||||
|
||||
#endif //DEMO_UTILS_H
|
||||
@@ -32,12 +32,14 @@ void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores,
|
||||
int *topk_inds, float *topk_ys, float *topk_xs);
|
||||
// void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes);
|
||||
void normalize(float *bgr, const int ch, const int h, const int w, const float *mean, const float *stddev);
|
||||
void transformDep(float *src_begin, float *src_end, float *dst_begin, float *dst_end);
|
||||
void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op);
|
||||
void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys);
|
||||
void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin,
|
||||
float *xs_begin, float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out);
|
||||
void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin,
|
||||
dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out);
|
||||
void getRecordsFromTopKId(int * ids_begin, const int K, const int ch, const int size, dnnType *src_begin, float *src_out, int *ids_out);
|
||||
|
||||
void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w);
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ public:
|
||||
height_ones = (i_h + 2 * ph - (1 * (kh - 1) + 1)) / sh + 1;
|
||||
width_ones = (i_w + 2 * pw - (1 * (kw - 1) + 1)) / sw + 1;
|
||||
dim_ones = i_c * kh * kw * 1 * height_ones * width_ones;
|
||||
std::cout<<i_c * o_c * kh * kw * 1<<"\n";
|
||||
|
||||
checkCuda( cudaMalloc(&data_d, i_c * o_c * kh * kw * 1 * sizeof(dnnType)));
|
||||
checkCuda( cudaMalloc(&bias2_d, o_c*sizeof(dnnType)));
|
||||
checkCuda( cudaMalloc(&ones_d1, height_ones * width_ones * sizeof(dnnType)));
|
||||
|
||||
@@ -12,8 +12,10 @@
|
||||
#include <cublas_v2.h>
|
||||
#include <cudnn.h>
|
||||
|
||||
|
||||
#ifdef __linux__
|
||||
#include <unistd.h>
|
||||
|
||||
#endif
|
||||
|
||||
#include <ios>
|
||||
|
||||
@@ -72,8 +72,6 @@ do
|
||||
# ./test_imuodom &>> $out_file
|
||||
# print_output $? imuodom
|
||||
|
||||
test_net shelfnet
|
||||
test_net shelfnet_berkeley
|
||||
test_net yolo4
|
||||
test_net yolo4_320
|
||||
test_net yolo4_320_coco2
|
||||
@@ -82,6 +80,7 @@ do
|
||||
test_net yolo4-csp
|
||||
test_net yolo4x
|
||||
test_net yolo4_berkeley
|
||||
test_net yolo4_berkeley_f1
|
||||
test_net yolo4tiny
|
||||
test_net yolo4tiny_512
|
||||
test_net yolo3
|
||||
@@ -98,9 +97,13 @@ do
|
||||
#test_net csresnext50-panet-spp_berkeley
|
||||
test_net resnet101_cnet
|
||||
test_net dla34_cnet
|
||||
test_net dla34_cnet3d
|
||||
test_net mobilenetv2ssd
|
||||
test_net mobilenetv2ssd512
|
||||
test_net bdd-mobilenetv2ssd
|
||||
test_net dla34_ctrack
|
||||
test_net shelfnet
|
||||
test_net shelfnet_berkeley
|
||||
done
|
||||
|
||||
echo "If errors occured, check logfile $out_file"
|
||||
|
||||
@@ -0,0 +1,892 @@
|
||||
#include "CenterTrack.h"
|
||||
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
|
||||
bool CenterTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches,
|
||||
const float conf_thresh, const bool mode_3d, const std::vector<cv::Mat>& k_calibs) {
|
||||
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
|
||||
dim = netRT->input_dim;
|
||||
dim.c = 3;
|
||||
nBatches = n_batches;
|
||||
confThreshold = conf_thresh;
|
||||
mode3D = mode_3d;
|
||||
inputCalibs = k_calibs;
|
||||
init_preprocessing();
|
||||
init_pre_inf();
|
||||
init_postprocessing();
|
||||
init_visualization(n_classes);
|
||||
}
|
||||
|
||||
bool CenterTrack::init_preprocessing(){
|
||||
//image transformation
|
||||
src = cv::Mat(cv::Size(2,3), CV_32F);
|
||||
dst = cv::Mat(cv::Size(2,3), CV_32F);
|
||||
dst2 = cv::Mat(cv::Size(2,3), CV_32F);
|
||||
trans = cv::Mat(cv::Size(3,2), CV_32F);
|
||||
trans2 = cv::Mat(cv::Size(3,2), CV_32F);
|
||||
transOut = cv::Mat(cv::Size(3,2), CV_32F);
|
||||
|
||||
dst2.at<float>(0,0) = width * 0.5;
|
||||
dst2.at<float>(0,1) = width * 0.5;
|
||||
dst2.at<float>(1,0) = width * 0.5;
|
||||
dst2.at<float>(1,1) = width * 0.5 + width * -0.5;
|
||||
dst2.at<float>(2,0) = dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
|
||||
dst2.at<float>(2,1) = dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
|
||||
|
||||
for(int bi=0; bi<nBatches; bi++) {
|
||||
szOld.push_back(cv::Size(0,0));
|
||||
}
|
||||
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
std::cout<<"OPENCV CPMTROB\n";
|
||||
checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) );
|
||||
float mean[3] = {0.40789655, 0.44719303, 0.47026116};
|
||||
float stddev[3] = {0.2886383, 0.27408165, 0.27809834};
|
||||
|
||||
checkCuda( cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
|
||||
checkCuda( cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
|
||||
#else
|
||||
std::cout<<"NO OPENCV CPMTROB\n";
|
||||
checkCuda( cudaMallocHost(&input, sizeof(dnnType)*dim.tot() * nBatches));
|
||||
mean << 0.40789655, 0.44719303, 0.47026116;
|
||||
stddev << 0.2886383, 0.27408165, 0.27809834;
|
||||
|
||||
#endif
|
||||
|
||||
checkCuda( cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
|
||||
checkCuda( cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot()));
|
||||
checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) );
|
||||
}
|
||||
|
||||
bool CenterTrack::init_pre_inf(){
|
||||
// initial steps: the first part of the network
|
||||
const char *pre_img_conv1_bin = "dla34_ctrack/layers/base-pre_img_layer-0.bin";
|
||||
const char *pre_hm_conv1_bin = "dla34_ctrack/layers/base-pre_hm_layer-0.bin";
|
||||
const char *conv1_bin = "dla34_ctrack/layers/base-base_layer-0.bin";
|
||||
const char *conv2_bin = "dla34_ctrack/layers/base-level0-0.bin";
|
||||
dim_in0 = tk::dnn::dataDim_t(1, 3, 512, 512, 1);
|
||||
dim_in1 = tk::dnn::dataDim_t(1, 1, 512, 512, 1);
|
||||
|
||||
checkCuda( cudaMalloc(&out_d, netRT->input_dim.tot()*sizeof(dnnType)) );
|
||||
checkCuda( cudaMalloc(&img_d, dim_in0.tot()*sizeof(dnnType)) );
|
||||
checkCuda( cudaMalloc(&hm_d, dim_in1.tot()*sizeof(dnnType)) );
|
||||
// init to zeros hm
|
||||
dnnType *hm_h;
|
||||
checkCuda( cudaMallocHost(&hm_h, 1 * dim.h * dim.w*sizeof(dnnType)) );
|
||||
for(int i=0; i<1 * dim.h * dim.w; i++)
|
||||
hm_h[i] = 0.0f;
|
||||
checkCuda( cudaMemcpy(hm_d, hm_h, 1 * dim.h * dim.w * sizeof(dnnType), cudaMemcpyHostToDevice) );
|
||||
checkCuda( cudaFreeHost(hm_h) );
|
||||
dnnType *i0_h, *i1_h, *i2_h;
|
||||
// dnnType *i0_d, *i1_d, *i2_d;
|
||||
|
||||
// const char *input_bin = "dla34_ctrack/debug/input.bin";
|
||||
// const char *pre_img_bin = "dla34_ctrack/debug/pre_imgages.bin";
|
||||
// const char *pre_hm_bin = "dla34_ctrack/debug/pre_hms.bin";
|
||||
// readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &img_d);
|
||||
// readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &hm_d);
|
||||
// readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &input_pre_inf_d);
|
||||
|
||||
pre_phase_net = new tk::dnn::Network(dim_in0);
|
||||
//pre-img
|
||||
tk::dnn::Input *in_pre_img = new tk::dnn::Input(pre_phase_net, dim_in0, img_d);
|
||||
tk::dnn::Conv2d *pre_img_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true);
|
||||
tk::dnn::Activation *pre_img_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU);
|
||||
//pre-hm
|
||||
tk::dnn::Input *in_pre_hm = new tk::dnn::Input(pre_phase_net, dim_in1, hm_d);
|
||||
tk::dnn::Conv2d *pre_hm_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true);
|
||||
tk::dnn::Activation *pre_hm_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU);
|
||||
// image input
|
||||
tk::dnn::Input *input_image = new tk::dnn::Input(pre_phase_net, dim_in0, input_pre_inf_d);
|
||||
tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true);
|
||||
tk::dnn::Activation *relu1 = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Shortcut *s0_input = new tk::dnn::Shortcut(pre_phase_net, pre_img_relu);
|
||||
tk::dnn::Shortcut *s1_input = new tk::dnn::Shortcut(pre_phase_net, pre_hm_relu);
|
||||
// output data
|
||||
out_d = s1_input->dstData;
|
||||
//print network model
|
||||
pre_phase_net->print();
|
||||
|
||||
iter0=true; // in the first iteration the last input is equal to the current input.
|
||||
return true;
|
||||
}
|
||||
|
||||
bool CenterTrack::init_postprocessing(){
|
||||
srand(0); //seed = 0 for random colors
|
||||
|
||||
dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1);
|
||||
dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
|
||||
dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
|
||||
dim_track = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
|
||||
dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1);
|
||||
dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1);
|
||||
dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
|
||||
dim_amodel_offset = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
|
||||
|
||||
checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) );
|
||||
checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
|
||||
checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
|
||||
for(int i=0; i<dim_hm.c * dim_hm.h * dim_hm.w; i++){
|
||||
ids_[i] = i;
|
||||
}
|
||||
|
||||
checkCuda( cudaMalloc(&ones, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
|
||||
float *ones_h;
|
||||
checkCuda( cudaMallocHost(&ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
|
||||
for(int i=0; i<dim_dep.c * dim_dep.h * dim_dep.w; i++)
|
||||
ones_h[i] = 1.0f;
|
||||
checkCuda( cudaMemcpy(ones, ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float), cudaMemcpyHostToDevice) );
|
||||
checkCuda( cudaFreeHost(ones_h) );
|
||||
|
||||
checkCuda( cudaMallocHost(&scores, K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&scores_d, K *sizeof(float)) );
|
||||
|
||||
checkCuda( cudaMallocHost(&clses, K *sizeof(int)) );
|
||||
checkCuda( cudaMalloc(&clses_d, K *sizeof(int)) );
|
||||
|
||||
checkCuda( cudaMalloc(&topk_inds_d, K *sizeof(int)) );
|
||||
checkCuda( cudaMalloc(&topk_ys_d, K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&topk_xs_d, K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&inttopk_ys_d, K *sizeof(int)) );
|
||||
checkCuda( cudaMalloc(&inttopk_xs_d, K *sizeof(int)) );
|
||||
|
||||
checkCuda( cudaMallocHost(&bbx0, K * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&bby0, K * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&bbx1, K * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&bby1, K * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&bbx0_d, K * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&bby0_d, K * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&bbx1_d, K * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&bby1_d, K * sizeof(float)) );
|
||||
|
||||
checkCuda( cudaMallocHost(&intxs, K * sizeof(int)) );
|
||||
checkCuda( cudaMallocHost(&intys, K * sizeof(int)) );
|
||||
|
||||
checkCuda( cudaMallocHost(&track, K * dim_track.c * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&dep, K * dim_dep.c * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&rot, K * dim_rot.c * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&dim_, K * dim_dim.c * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&wh, K * dim_wh.c * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&amodel_offset, K * dim_amodel_offset.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&track_d, K * dim_track.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&dep_d, K * dim_dep.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&rot_d, K * dim_rot.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&dim_d, K * dim_dim.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&wh_d, K * dim_wh.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&amodel_offset_d, K * dim_amodel_offset.c * sizeof(float)) );
|
||||
|
||||
checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) );
|
||||
|
||||
for(int bi=0; bi<nBatches; bi++) {
|
||||
cv::Mat calibs_ = cv::Mat::zeros(cv::Size(4,3), CV_32F);
|
||||
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
|
||||
calibs_.at<float>(0,0) = 633.0;
|
||||
calibs_.at<float>(1,1) = 633.0;
|
||||
calibs_.at<float>(2,2) = 1.0;
|
||||
}
|
||||
calibs_.at<float>(2,2) = 1.0;
|
||||
calibs.push_back(calibs_);
|
||||
}
|
||||
|
||||
// Alloc array used in the kernel
|
||||
checkCuda( cudaMalloc(&src_out, K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) );
|
||||
|
||||
trRes.resize(nBatches);
|
||||
countTr.resize(nBatches, 0);
|
||||
trackId.resize(nBatches, 0);
|
||||
}
|
||||
|
||||
bool CenterTrack::init_visualization(const int n_classes){
|
||||
classes = n_classes;
|
||||
// const char *kitti_class_name[] = {
|
||||
// "person", "car", "bicycle"};
|
||||
// classesNames = std::vector<std::string>(kitti_class_name, std::end( kitti_class_name));
|
||||
|
||||
const char *class_name[] = {"car", "truck", "bus", "trailer", "construction_vehicle", "pedestrian",
|
||||
"motorcycle", "bicycle", "traffic_cone", "barrier"};
|
||||
classesNames = std::vector<std::string>(class_name, std::end( class_name));
|
||||
|
||||
// const char *coco_class_name[] = {
|
||||
// "person", "bicycle", "car", "motorcycle", "airplane",
|
||||
// "bus", "train", "truck", "boat", "traffic light", "fire hydrant",
|
||||
// "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse",
|
||||
// "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
|
||||
// "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis",
|
||||
// "snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
|
||||
// "skateboard", "surfboard", "tennis racket", "bottle", "wine glass",
|
||||
// "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich",
|
||||
// "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake",
|
||||
// "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv",
|
||||
// "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
|
||||
// "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
|
||||
// "scissors", "teddy bear", "hair drier", "toothbrush"
|
||||
// };
|
||||
// classesNames = std::vector<std::string>(coco_class_name, std::end( coco_class_name));
|
||||
|
||||
for(int c=0; c<classes; c++) {
|
||||
int offset = c*123457 % classes;
|
||||
float r = getColor(2, offset, classes);
|
||||
float g = getColor(1, offset, classes);
|
||||
float b = getColor(0, offset, classes);
|
||||
colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
|
||||
}
|
||||
for(int c=0; c<256; c++) {
|
||||
int offset = c * 123457 % 256;
|
||||
float r = getColor(2, offset, 256);
|
||||
float g = getColor(1, offset, 256);
|
||||
float b = getColor(0, offset, 256);
|
||||
trColors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
|
||||
}
|
||||
|
||||
r = cv::Mat(cv::Size(3,3), CV_32F);
|
||||
r.at<float>(0,1) = 0.0;
|
||||
r.at<float>(1,0) = 0.0;
|
||||
r.at<float>(1,1) = 1.0;
|
||||
r.at<float>(1,2) = 0.0;
|
||||
r.at<float>(2,1) = 0.0;
|
||||
|
||||
corners = cv::Mat(cv::Size(8,3), CV_32F);
|
||||
corners.at<float>(1,0) = 0.0;
|
||||
corners.at<float>(1,1) = 0.0;
|
||||
corners.at<float>(1,2) = 0.0;
|
||||
corners.at<float>(1,3) = 0.0;
|
||||
|
||||
pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F);
|
||||
pts3DHomo.at<float>(3,0) = 1.0;
|
||||
pts3DHomo.at<float>(3,1) = 1.0;
|
||||
pts3DHomo.at<float>(3,2) = 1.0;
|
||||
pts3DHomo.at<float>(3,3) = 1.0;
|
||||
pts3DHomo.at<float>(3,4) = 1.0;
|
||||
pts3DHomo.at<float>(3,5) = 1.0;
|
||||
pts3DHomo.at<float>(3,6) = 1.0;
|
||||
pts3DHomo.at<float>(3,7) = 1.0;
|
||||
|
||||
faceId.push_back({0,1,5,4});
|
||||
faceId.push_back({1,2,6, 5});
|
||||
faceId.push_back({3,0,4,7});
|
||||
faceId.push_back({2,3,7,6});
|
||||
// ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]);
|
||||
}
|
||||
|
||||
void CenterTrack::_get_additional_inputs(){
|
||||
//None no additional input
|
||||
}
|
||||
|
||||
void CenterTrack::pre_inf(const int bi){
|
||||
TKDNN_TSTART
|
||||
tk::dnn::dataDim_t dim_aus;
|
||||
pre_phase_net->infer(dim_aus, nullptr);
|
||||
TKDNN_TSTOP
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
checkCuda( cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
}
|
||||
|
||||
void CenterTrack::preprocess(cv::Mat &frame, const int bi){
|
||||
cv::Size sz = originalSize[bi];
|
||||
// float scale = 1.0;
|
||||
float new_height = dim.h;//sz.height * scale;
|
||||
float new_width = dim.w;//sz.width * scale;
|
||||
if(sz.height != szOld[bi].height && sz.width != szOld[bi].width){
|
||||
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
|
||||
calibs[bi].at<float>(0,2) = new_width / 2.0f;
|
||||
calibs[bi].at<float>(1,2) = new_height /2.0f;
|
||||
}
|
||||
else {
|
||||
calibs[bi].at<float>(0,0) = inputCalibs[bi].at<float>(0,0) * dim.w / sz.width;
|
||||
calibs[bi].at<float>(0,2) = inputCalibs[bi].at<float>(0,2) * dim.w / sz.width;
|
||||
calibs[bi].at<float>(1,1) = inputCalibs[bi].at<float>(1,1) * dim.h / sz.height;
|
||||
calibs[bi].at<float>(1,2) = inputCalibs[bi].at<float>(1,2) * dim.h / sz.height;
|
||||
}
|
||||
|
||||
float c[] = {new_width / 2.0f, new_height /2.0f};
|
||||
float s[] = {dim.w, dim.h};
|
||||
// float s = new_width >= new_height ? new_width : new_height;
|
||||
// ----------- get_affine_transform
|
||||
// rot_rad = pi * 0 / 100 --> 0
|
||||
//dim.print();
|
||||
src.at<float>(0,0) = c[0];
|
||||
src.at<float>(0,1) = c[1];
|
||||
src.at<float>(1,0) = c[0];
|
||||
src.at<float>(1,1) = c[1] + s[0] * -0.5;
|
||||
dst.at<float>(0,0) = dim.w * 0.5;
|
||||
dst.at<float>(0,1) = dim.h * 0.5;
|
||||
dst.at<float>(1,0) = dim.w * 0.5;
|
||||
dst.at<float>(1,1) = dim.h * 0.5 + dim.w * -0.5;
|
||||
|
||||
src.at<float>(2,0) = src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
|
||||
src.at<float>(2,1) = src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
|
||||
dst.at<float>(2,0) = dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
|
||||
dst.at<float>(2,1) = dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
|
||||
|
||||
|
||||
trans = cv::getAffineTransform( src, dst );
|
||||
trans2 = cv::getAffineTransform( dst2, src );
|
||||
trans2.convertTo(transOut, CV_32F);
|
||||
}
|
||||
szOld[bi] = sz;
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
cv::cuda::GpuMat im_Orig;
|
||||
cv::cuda::GpuMat imageF1_d, imageF2_d;
|
||||
|
||||
im_Orig = cv::cuda::GpuMat(frame);
|
||||
cv::cuda::resize (im_Orig, imageF1_d, cv::Size(dim.w, dim.h));
|
||||
// imageF1_d = im_Orig;
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
sz = imageF1_d.size();
|
||||
|
||||
cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR );
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0);
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
dim2 = dim;
|
||||
cv::cuda::GpuMat bgr[3];
|
||||
cv::cuda::split(imageF1_d,bgr);//split source
|
||||
|
||||
for(int i=0; i<dim.c; i++)
|
||||
checkCuda( cudaMemcpy(d_ptrs + i*dim.h * dim.w, (float*)bgr[i].data, dim.h * dim.w * sizeof(float), cudaMemcpyDeviceToDevice) );
|
||||
|
||||
normalize(d_ptrs, dim.c, dim.h, dim.w, mean_d, stddev_d);
|
||||
|
||||
checkCuda( cudaMemcpy(input_pre_inf_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
#else
|
||||
cv::Mat imageF;
|
||||
resize(frame, imageF, cv::Size(dim.w, dim.h));
|
||||
// imageF = frame;
|
||||
sz = imageF.size();
|
||||
cv::warpAffine(imageF, imageF, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR );
|
||||
|
||||
// cv::imshow("warp", imageF);
|
||||
|
||||
sz = imageF.size();
|
||||
imageF.convertTo(imageF, CV_32FC3, 1/255.0);
|
||||
|
||||
dim2 = dim;
|
||||
//split channels
|
||||
cv::Mat bgr[3];
|
||||
cv::split(imageF,bgr);//split source
|
||||
|
||||
for(int i=0; i<3; i++){
|
||||
bgr[i] = bgr[i] - mean[i];
|
||||
bgr[i] = bgr[i] / stddev[i];
|
||||
}
|
||||
for(int i=0; i<dim2.c; i++) {
|
||||
int idx = i * imageF.rows * imageF.cols;
|
||||
int ch = i;
|
||||
memcpy((void*)&input[idx], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
|
||||
}
|
||||
checkCuda( cudaMemcpyAsync(input_pre_inf_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
#endif
|
||||
|
||||
if(iter0) {
|
||||
checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
iter0=false;
|
||||
}
|
||||
pre_inf(bi);
|
||||
|
||||
checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
}
|
||||
|
||||
cv::Mat CenterTrack::transform_preds_with_trans(float x1, float x2){
|
||||
cv::Mat target_coords(cv::Size(1,3), CV_32F);
|
||||
target_coords.at<float>(0,0) = x1;
|
||||
target_coords.at<float>(0,1) = x2;
|
||||
target_coords.at<float>(0,2) = 1.0;
|
||||
return transOut * target_coords;
|
||||
}
|
||||
|
||||
void CenterTrack::tracking(const int bi) {
|
||||
float item_size[countDet];
|
||||
int item_cl[countDet];
|
||||
float dets[2*countDet];
|
||||
for(int i=0; i<countDet; i++){
|
||||
item_size[i] = (detRes[i].bb1.at<float>(0,0) - detRes[i].bb0.at<float>(0,0)) *
|
||||
(detRes[i].bb1.at<float>(0,1) - detRes[i].bb0.at<float>(0,1));
|
||||
item_cl[i] = detRes[i].cl;
|
||||
dets[i*2] = detRes[i].ct.at<float>(0,0);
|
||||
dets[i*2+1] = detRes[i].ct.at<float>(0,1);
|
||||
}
|
||||
|
||||
float track_size[countTr[bi]];
|
||||
int track_cl[countTr[bi]];
|
||||
float tracks[2*countTr[bi]];
|
||||
for(int i=0; i<countTr[bi]; i++){
|
||||
track_size[i] = (trRes[bi][i].det_res.bb1.at<float>(0,0) - trRes[bi][i].det_res.bb0.at<float>(0,0)) *
|
||||
(trRes[bi][i].det_res.bb1.at<float>(0,1) - trRes[bi][i].det_res.bb0.at<float>(0,1));
|
||||
track_cl[i] = trRes[bi][i].det_res.cl;
|
||||
tracks[i*2] = trRes[bi][i].det_res.ct.at<float>(0,0);
|
||||
tracks[i*2+1] = trRes[bi][i].det_res.ct.at<float>(0,1);
|
||||
}
|
||||
float dist[countTr[bi]*countDet];
|
||||
bool invalid;
|
||||
for(int i=0; i<countTr[bi]; i++){
|
||||
for(int j=0; j<countDet; j++){
|
||||
dist[j*countTr[bi]+i] = pow((tracks[i*2] - dets[j*2]), 2) +
|
||||
pow((tracks[i*2+1] - dets[j*2+1]), 2);
|
||||
invalid = dist[j*countTr[bi]+i] > track_size[i] ||
|
||||
dist[j*countTr[bi]+i] > item_size[j] ||
|
||||
item_cl[j] != track_cl[i];
|
||||
dist[j*countTr[bi]+i] = dist[j*countTr[bi]+i] + invalid * (1 << 18);
|
||||
}
|
||||
}
|
||||
int matched_indices[2*countTr[bi]];
|
||||
float min_tr;
|
||||
int min_idtr = -1;
|
||||
for(int i=0; i<countTr[bi]; i++) {
|
||||
matched_indices[i*2] = -1;
|
||||
matched_indices[i*2+1] = -1;
|
||||
}
|
||||
for(int i=0; i<countDet; i++){
|
||||
min_tr=(1 << 18);
|
||||
for(int j=0; j<countTr[bi]; j++){
|
||||
if(dist[i*countTr[bi]+j]<min_tr) {
|
||||
min_tr = dist[i*countTr[bi]+j];
|
||||
min_idtr = j;
|
||||
}
|
||||
}
|
||||
if(min_tr < (1<<16)) {
|
||||
for(int j=0; j<countDet; j++)
|
||||
dist[j*countTr[bi]+min_idtr] = (1 << 18);
|
||||
matched_indices[2*min_idtr] = min_idtr;
|
||||
matched_indices[2*min_idtr+1] = i;
|
||||
}
|
||||
}
|
||||
|
||||
bool unmatched_dets[countDet];
|
||||
for(int i=0; i<countDet; i++)
|
||||
unmatched_dets[i] = false;
|
||||
bool unmatched_tracks[countTr[bi]];
|
||||
for(int i=0; i<countTr[bi]; i++)
|
||||
unmatched_tracks[i] = false;
|
||||
for(int i=0; i<countTr[bi]; i++) {
|
||||
if(matched_indices[2*i] != -1)
|
||||
unmatched_tracks[matched_indices[2*i]]=true;
|
||||
if(matched_indices[2*i+1] != -1)
|
||||
unmatched_dets[matched_indices[2*i+1]]=true;
|
||||
}
|
||||
|
||||
//match
|
||||
for(int i=0; i<countTr[bi]; i++) {
|
||||
if(matched_indices[2*i+1] != -1 && matched_indices[2*i] != -1) { //second condition is optional
|
||||
int tr_id = matched_indices[2*i];
|
||||
int d_id = matched_indices[2*i+1];
|
||||
|
||||
// trRes[tr_id].det_res = detRes[d_id];
|
||||
trRes[bi][tr_id].det_res.score = detRes[d_id].score;
|
||||
trRes[bi][tr_id].det_res.cl = detRes[d_id].cl;
|
||||
trRes[bi][tr_id].det_res.ct = detRes[d_id].ct;
|
||||
trRes[bi][tr_id].det_res.tr = detRes[d_id].tr;
|
||||
trRes[bi][tr_id].det_res.bb0 = detRes[d_id].bb0;
|
||||
trRes[bi][tr_id].det_res.bb1 = detRes[d_id].bb1;
|
||||
trRes[bi][tr_id].det_res.dep = detRes[d_id].dep;
|
||||
trRes[bi][tr_id].det_res.dim[0] = detRes[d_id].dim[0];
|
||||
trRes[bi][tr_id].det_res.dim[1] = detRes[d_id].dim[1];
|
||||
trRes[bi][tr_id].det_res.dim[2] = detRes[d_id].dim[2];
|
||||
trRes[bi][tr_id].det_res.alpha = detRes[d_id].alpha;
|
||||
trRes[bi][tr_id].det_res.x = detRes[d_id].x;
|
||||
trRes[bi][tr_id].det_res.y = detRes[d_id].y;
|
||||
trRes[bi][tr_id].det_res.z = detRes[d_id].z;
|
||||
trRes[bi][tr_id].det_res.rot_y = detRes[d_id].rot_y;
|
||||
// trRes[bi][matched_indices[2*i]].tracking_id = ; is the same
|
||||
// trRes[bi][matched_indices[2*i]].color = ; is the same
|
||||
trRes[bi][tr_id].age = 1;
|
||||
trRes[bi][tr_id].active = trRes[bi][tr_id].active+1;
|
||||
}
|
||||
}
|
||||
//delete target umatched track
|
||||
int new_count_tr = 0;
|
||||
for(int i=0; i<countTr[bi]; i++) {
|
||||
if(unmatched_tracks[i])
|
||||
new_count_tr++;
|
||||
}
|
||||
if(new_count_tr == 0 && countTr[bi] != 0) { //reset
|
||||
trRes[bi].clear();
|
||||
countTr[bi] = 0;
|
||||
}
|
||||
int old_count_tr = countTr[bi];
|
||||
if(countTr[bi] != 0 && new_count_tr != countTr[bi]) {
|
||||
std::vector<struct trackingRes> new_tr_res;
|
||||
int id_new_tr=0;
|
||||
for(int i=0; i<countTr[bi]; i++) {
|
||||
if(unmatched_tracks[i]) {
|
||||
struct trackingRes new_tr_res_;
|
||||
// new_tr_res_new_det_res.det_res = trRes[i].det_res;
|
||||
new_tr_res_.det_res.score = trRes[bi][i].det_res.score;
|
||||
new_tr_res_.det_res.cl = trRes[bi][i].det_res.cl;
|
||||
new_tr_res_.det_res.ct = trRes[bi][i].det_res.ct;
|
||||
new_tr_res_.det_res.tr = trRes[bi][i].det_res.tr;
|
||||
new_tr_res_.det_res.bb0 = trRes[bi][i].det_res.bb0;
|
||||
new_tr_res_.det_res.bb1 = trRes[bi][i].det_res.bb1;
|
||||
new_tr_res_.det_res.dep = trRes[bi][i].det_res.dep;
|
||||
new_tr_res_.det_res.dim[0] = trRes[bi][i].det_res.dim[0];
|
||||
new_tr_res_.det_res.dim[1] = trRes[bi][i].det_res.dim[1];
|
||||
new_tr_res_.det_res.dim[2] = trRes[bi][i].det_res.dim[2];
|
||||
new_tr_res_.det_res.alpha = trRes[bi][i].det_res.alpha;
|
||||
new_tr_res_.det_res.x = trRes[bi][i].det_res.x;
|
||||
new_tr_res_.det_res.y = trRes[bi][i].det_res.y;
|
||||
new_tr_res_.det_res.z = trRes[bi][i].det_res.z;
|
||||
new_tr_res_.det_res.rot_y = trRes[bi][i].det_res.rot_y;
|
||||
new_tr_res_.tracking_id = trRes[bi][i].tracking_id;
|
||||
new_tr_res_.age = trRes[bi][i].age;
|
||||
new_tr_res_.active = trRes[bi][i].active;
|
||||
new_tr_res_.color = trRes[bi][i].color;
|
||||
id_new_tr ++;
|
||||
new_tr_res.push_back(new_tr_res_);
|
||||
}
|
||||
}
|
||||
|
||||
if(countTr[bi]) {
|
||||
trRes[bi].clear();
|
||||
}
|
||||
countTr[bi] = new_count_tr;
|
||||
trRes[bi] = new_tr_res;
|
||||
}
|
||||
|
||||
int count_tr_ = countTr[bi];
|
||||
for(int i=0; i<countDet; i++) {
|
||||
if((!unmatched_dets[i]) && detRes[i].score > newThresh) {
|
||||
count_tr_ ++;
|
||||
struct trackingRes new_tr_res_;
|
||||
new_tr_res_.det_res.score = detRes[i].score;
|
||||
new_tr_res_.det_res.cl = detRes[i].cl;
|
||||
new_tr_res_.det_res.ct = detRes[i].ct;
|
||||
new_tr_res_.det_res.tr = detRes[i].tr;
|
||||
new_tr_res_.det_res.bb0 = detRes[i].bb0;
|
||||
new_tr_res_.det_res.bb1 = detRes[i].bb1;
|
||||
new_tr_res_.det_res.dep = detRes[i].dep;
|
||||
new_tr_res_.det_res.dim[0] = detRes[i].dim[0];
|
||||
new_tr_res_.det_res.dim[1] = detRes[i].dim[1];
|
||||
new_tr_res_.det_res.dim[2] = detRes[i].dim[2];
|
||||
new_tr_res_.det_res.alpha = detRes[i].alpha;
|
||||
new_tr_res_.det_res.x = detRes[i].x;
|
||||
new_tr_res_.det_res.y = detRes[i].y;
|
||||
new_tr_res_.det_res.z = detRes[i].z;
|
||||
new_tr_res_.det_res.rot_y = detRes[i].rot_y;
|
||||
new_tr_res_.tracking_id = trackId[bi]++;
|
||||
new_tr_res_.age = 1;
|
||||
new_tr_res_.active = 1;
|
||||
new_tr_res_.color = rand() % 256;
|
||||
if(trRes.size() <= bi) {
|
||||
std::vector<struct trackingRes> v_new_tr_res_;
|
||||
v_new_tr_res_.push_back(new_tr_res_);
|
||||
trRes.push_back(v_new_tr_res_);
|
||||
}
|
||||
else
|
||||
trRes[bi].push_back(new_tr_res_);
|
||||
}
|
||||
}
|
||||
|
||||
countTr[bi] = count_tr_;
|
||||
//reset the tracker id
|
||||
if(trackId[bi] == 1000)
|
||||
trackId[bi] = 0;
|
||||
detRes.clear();
|
||||
|
||||
}
|
||||
|
||||
void CenterTrack::postprocess(const int bi, const bool mAP) {
|
||||
dnnType *rt_out[9];
|
||||
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
|
||||
rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
|
||||
rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
|
||||
rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
|
||||
rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi;
|
||||
rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi;
|
||||
rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi;
|
||||
rt_out[7] = (dnnType *)netRT->buffersRT[8]+ netRT->buffersDIM[8].tot()*bi;
|
||||
rt_out[8] = (dnnType *)netRT->buffersRT[9]+ netRT->buffersDIM[9].tot()*bi;
|
||||
|
||||
// ------------------------------------ process --------------------------------------------
|
||||
|
||||
activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot());
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
// output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1.
|
||||
activationSIGMOIDForward(rt_out[5], rt_out[5], dim_dep.tot());
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
transformDep(ones, ones + dim_dep.tot(), rt_out[5], rt_out[5] + dim_dep.tot());
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
// nms
|
||||
subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op);
|
||||
|
||||
// ----------- nms end
|
||||
// ----------- topk
|
||||
|
||||
if(K > dim_hm.h * dim_hm.w){
|
||||
printf ("Error topk (K is too large)\n");
|
||||
return;
|
||||
}
|
||||
|
||||
checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );
|
||||
|
||||
sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d);
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d);
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d);
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
|
||||
checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
|
||||
|
||||
checkCuda( cudaMemcpy(intxs, inttopk_xs_d, K * sizeof(int), cudaMemcpyDeviceToHost) );
|
||||
checkCuda( cudaMemcpy(intys, inttopk_ys_d, K * sizeof(int), cudaMemcpyDeviceToHost) );
|
||||
|
||||
checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
|
||||
|
||||
// ----------- topk end
|
||||
|
||||
topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out);
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out);
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
checkCuda( cudaMemcpy(bby0, bby0_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
checkCuda( cudaMemcpy(bbx1, bbx1_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
//regression heads
|
||||
// ['tracking', 'dep', 'rot', 'dim', 'amodel_offset',
|
||||
// 'nuscenes_att', 'velocity']
|
||||
getRecordsFromTopKId(topk_inds_d, K, dim_track.c, dim_track.h * dim_track.w, rt_out[4], track_d, ids_out);
|
||||
checkCuda( cudaMemcpy(track, track_d, K * dim_track.c * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[5], dep_d, ids_out);
|
||||
checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[6], rot_d, ids_out);
|
||||
checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[7], dim_d, ids_out);
|
||||
checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
getRecordsFromTopKId(topk_inds_d, K, dim_amodel_offset.c, dim_amodel_offset.h * dim_amodel_offset.w, rt_out[8], amodel_offset_d, ids_out);
|
||||
checkCuda( cudaMemcpy(amodel_offset, amodel_offset_d, K * dim_amodel_offset.c * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
// ---------------------------------- post-process -----------------------------------------
|
||||
|
||||
countDet = 0;
|
||||
detRes.clear();
|
||||
for(int i=0; i<K; i++){
|
||||
if(scores[i] < outThresh)
|
||||
break;
|
||||
|
||||
countDet ++;
|
||||
struct detectionRes new_det_res;
|
||||
new_det_res.score = scores[i];
|
||||
new_det_res.cl = clses[i]+1;
|
||||
// ret_s=scores[i];
|
||||
// ret_c=clses[i]+1;
|
||||
new_det_res.ct = transform_preds_with_trans(intxs[i], intys[i]);
|
||||
new_det_res.tr = transform_preds_with_trans(intxs[i] + track[i], intys[i] + track[i+K]);
|
||||
new_det_res.tr = new_det_res.tr -new_det_res.ct;
|
||||
new_det_res.bb0 = transform_preds_with_trans(bbx0[i], bby0[i]);
|
||||
new_det_res.bb1 = transform_preds_with_trans(bbx1[i], bby1[i]);
|
||||
new_det_res.ct = transform_preds_with_trans(((bbx0[i]+bbx1[i])/2 + amodel_offset[i]),
|
||||
((bby0[i]+bby1[i])/2 + amodel_offset[i+K]));
|
||||
new_det_res.dep = dep[i];
|
||||
new_det_res.dim[0] = dim_[i];
|
||||
new_det_res.dim[1] = dim_[i+K];
|
||||
new_det_res.dim[2] = dim_[i+2*K];
|
||||
|
||||
// unproject_2d_to_3d
|
||||
new_det_res.z = dep[i] - calibs[bi].at<float>(2,3);
|
||||
new_det_res.x = ((float)new_det_res.ct.at<float>(0,0) * dep[i] - calibs[bi].at<float>(0,3) -
|
||||
calibs[bi].at<float>(0,2) * new_det_res.z) / calibs[bi].at<float>(0,0);
|
||||
new_det_res.y = ((float)new_det_res.ct.at<float>(0,1) * dep[i] - calibs[bi].at<float>(1,3) -
|
||||
calibs[bi].at<float>(1,2) * new_det_res.z) / calibs[bi].at<float>(1,1) + (dim_[i] / 2);
|
||||
|
||||
// alpha2rot_y
|
||||
// idx = rot[:, 1] > rot[:, 5]
|
||||
// alpha1 = np.arctan2(rot[:, 2], rot[:, 3]) + (-0.5 * np.pi)
|
||||
// alpha2 = np.arctan2(rot[:, 6], rot[:, 7]) + ( 0.5 * np.pi)
|
||||
// return alpha1 * idx + alpha2 * (1 - idx)
|
||||
if(rot[1*K + i] > rot[5*K + i])
|
||||
new_det_res.alpha = std::atan2(rot[2*K + i], rot[3*K + i]) -0.5 * M_PI;
|
||||
else
|
||||
new_det_res.alpha = std::atan2(rot[6*K + i], rot[7*K + i]) +0.5 * M_PI;
|
||||
new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at<float>(0,0) - calibs[bi].at<float>(0,2), calibs[bi].at<float>(0,0)));
|
||||
new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest
|
||||
detRes.push_back(new_det_res);
|
||||
}
|
||||
// track step
|
||||
tracking(bi);
|
||||
}
|
||||
|
||||
void CenterTrack::draw(std::vector<cv::Mat>& frames) {
|
||||
struct trackingRes t;
|
||||
float sc;
|
||||
int id;
|
||||
std::string txt;
|
||||
int baseline = 0;
|
||||
float font_scale = 0.8;
|
||||
int thickness = 2;
|
||||
|
||||
for(int bi=0; bi<frames.size(); ++bi) {
|
||||
float scale_x = float(originalSize[bi].width)/dim.w;
|
||||
float scale_y = float(originalSize[bi].height)/dim.h;
|
||||
resize(frames[bi], frames[bi], originalSize[bi]);
|
||||
// draw dets
|
||||
for(int i=0; trRes.size() != 0 && i<trRes[bi].size(); i++) {
|
||||
t = trRes[bi][i];
|
||||
id = t.tracking_id;
|
||||
txt = classesNames[t.det_res.cl-1]+'-'+std::to_string(id); //forse ha bisogno di cl-1
|
||||
cv::Size text_size = getTextSize(txt, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
|
||||
|
||||
if(t.det_res.score > confThreshold){// && t.active!=0) {
|
||||
if(!mode3D) {
|
||||
cv::rectangle(frames[bi],
|
||||
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x, t.det_res.bb0.at<float>(0,1) * scale_y),
|
||||
cv::Point(t.det_res.bb1.at<float>(0,0) * scale_x, t.det_res.bb1.at<float>(0,1) * scale_y),
|
||||
trColors[t.color], thickness);
|
||||
cv::rectangle(frames[bi],
|
||||
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x, t.det_res.bb0.at<float>(0,1) * scale_y - text_size.height - thickness),
|
||||
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x + text_size.width, t.det_res.bb0.at<float>(0,1) * scale_y),
|
||||
trColors[t.color], -1);
|
||||
|
||||
cv::putText(frames[bi], txt,
|
||||
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x, t.det_res.bb0.at<float>(0,1) * scale_y - thickness -1),
|
||||
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1);
|
||||
|
||||
cv::arrowedLine(frames[bi],
|
||||
cv::Point((int)t.det_res.ct.at<float>(0,0) * scale_x, (int)t.det_res.ct.at<float>(0,1) * scale_y),
|
||||
cv::Point((int)(t.det_res.ct.at<float>(0,0) * scale_x + t.det_res.tr.at<float>(0,0) * scale_x),
|
||||
(int)(t.det_res.ct.at<float>(0,1) * scale_y + t.det_res.tr.at<float>(0,1) * scale_y)),
|
||||
cv::Scalar(255, 0, 255), 2);
|
||||
}
|
||||
//3d
|
||||
if(mode3D && t.det_res.z > 1){
|
||||
r.at<float>(0,0) = std::cos(t.det_res.rot_y);
|
||||
r.at<float>(0,2) = std::sin(t.det_res.rot_y);
|
||||
r.at<float>(2,0) = -std::sin(t.det_res.rot_y);
|
||||
r.at<float>(2,2) = std::cos(t.det_res.rot_y);
|
||||
|
||||
corners.at<float>(0,0) = t.det_res.dim[2]/2;
|
||||
corners.at<float>(0,1) = t.det_res.dim[2]/2;
|
||||
corners.at<float>(0,2) = -t.det_res.dim[2]/2;
|
||||
corners.at<float>(0,3) = -t.det_res.dim[2]/2;
|
||||
corners.at<float>(0,4) = t.det_res.dim[2]/2;
|
||||
corners.at<float>(0,5) = t.det_res.dim[2]/2;
|
||||
corners.at<float>(0,6) = -t.det_res.dim[2]/2;
|
||||
corners.at<float>(0,7) = -t.det_res.dim[2]/2;
|
||||
|
||||
corners.at<float>(1,4) = -t.det_res.dim[0];
|
||||
corners.at<float>(1,5) = -t.det_res.dim[0];
|
||||
corners.at<float>(1,6) = -t.det_res.dim[0];
|
||||
corners.at<float>(1,7) = -t.det_res.dim[0];
|
||||
|
||||
corners.at<float>(2,0) = t.det_res.dim[1]/2;
|
||||
corners.at<float>(2,1) = -t.det_res.dim[1]/2;
|
||||
corners.at<float>(2,2) = -t.det_res.dim[1]/2;
|
||||
corners.at<float>(2,3) = t.det_res.dim[1]/2;
|
||||
corners.at<float>(2,4) = t.det_res.dim[1]/2;
|
||||
corners.at<float>(2,5) = -t.det_res.dim[1]/2;
|
||||
corners.at<float>(2,6) = -t.det_res.dim[1]/2;
|
||||
corners.at<float>(2,7) = t.det_res.dim[1]/2;
|
||||
|
||||
cv::Mat aus = r * corners;
|
||||
|
||||
for(int k=0; k<8; k++) {
|
||||
aus.at<float>(0,k) += t.det_res.x;
|
||||
aus.at<float>(1,k) += t.det_res.y;
|
||||
aus.at<float>(2,k) += t.det_res.z;
|
||||
}
|
||||
|
||||
// corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3)));
|
||||
for(int k1=0; k1<3; k1++) {
|
||||
for(int k2=0; k2<8; k2++)
|
||||
pts3DHomo.at<float>(k1,k2) = aus.at<float>(k1,k2);
|
||||
}
|
||||
|
||||
aus.release();
|
||||
aus = calibs[bi] * pts3DHomo;
|
||||
std::vector<float> res_corners;
|
||||
for(int k=0; k<8; k++) {
|
||||
res_corners.push_back(aus.at<float>(0,k) / aus.at<float>(2,k));
|
||||
res_corners.push_back(aus.at<float>(1,k) / aus.at<float>(2,k));
|
||||
}
|
||||
aus.release();
|
||||
for(int ind_f=3; ind_f>=0; ind_f--) {
|
||||
for(int j=0; j<4; j++) {
|
||||
cv::line(frames[bi],
|
||||
cv::Point((int)res_corners.at(faceId.at(ind_f).at(j) * 2) * scale_x,
|
||||
(int)res_corners.at(faceId.at(ind_f).at(j) * 2 + 1) * scale_y),
|
||||
cv::Point((int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2) * scale_x,
|
||||
(int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1) * scale_y),
|
||||
trColors[t.color], 2);
|
||||
if(ind_f == 0 && j==3) {
|
||||
cv::line(frames[bi],
|
||||
cv::Point((int)res_corners.at(faceId.at(ind_f).at(0) * 2) * scale_x,
|
||||
(int)res_corners.at(faceId.at(ind_f).at(0) * 2 + 1) * scale_y),
|
||||
cv::Point((int)res_corners.at(faceId.at(ind_f).at(2) * 2) * scale_x,
|
||||
(int)res_corners.at(faceId.at(ind_f).at(2) * 2 + 1) * scale_y), trColors[t.color], 2);
|
||||
cv::line(frames[bi],
|
||||
cv::Point((int)res_corners.at(faceId.at(ind_f).at(1) * 2) * scale_x,
|
||||
(int)res_corners.at(faceId.at(ind_f).at(1) * 2 + 1) * scale_y),
|
||||
cv::Point((int)res_corners.at(faceId.at(ind_f).at(3) * 2) * scale_x,
|
||||
(int)res_corners.at(faceId.at(ind_f).at(3) * 2 + 1) * scale_y), trColors[t.color], 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0;
|
||||
for(int k=0; k<8; k++) {
|
||||
if(res_corners[2*k] < bb0)
|
||||
bb0 = res_corners[2*k];
|
||||
if(res_corners[2*k] > bb1)
|
||||
bb1 = res_corners[2*k];
|
||||
if(res_corners[2*k+1] < bb2)
|
||||
bb2 = res_corners[2*k+1];
|
||||
if(res_corners[2*k+1] > bb3)
|
||||
bb3 = res_corners[2*k+1];
|
||||
|
||||
}
|
||||
// if(not no_bbox):
|
||||
// cv::rectangle(frame,
|
||||
// cv::Point(bb0, bb2),
|
||||
// cv::Point(bb1, bb3),
|
||||
// trColors[t.color], thickness);
|
||||
cv::rectangle(frames[bi],
|
||||
cv::Point(bb0 * scale_x, bb2 * scale_y - text_size.height - thickness),
|
||||
cv::Point(bb0 * scale_x + text_size.width, bb2 * scale_y),
|
||||
trColors[t.color], -1);
|
||||
|
||||
cv::putText(frames[bi], txt,
|
||||
cv::Point(bb0 * scale_x, bb2 * scale_y - thickness -1),
|
||||
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1);
|
||||
|
||||
cv::arrowedLine(frames[bi],
|
||||
cv::Point((int)((bb0 + bb1)/2) * scale_x, (int)((bb2 + bb3)/2) * scale_y),
|
||||
cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at<float>(0,0)) * scale_x,
|
||||
(int)((bb2 + bb3)/2 + t.det_res.tr.at<float>(0,1)) * scale_y),
|
||||
cv::Scalar(255, 0, 255), 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}}
|
||||
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
#include "CenternetDetection3D.h"
|
||||
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches,
|
||||
const float conf_thresh, const std::vector<cv::Mat>& k_calibs) {
|
||||
std::cout<<(tensor_path).c_str()<<"\n";
|
||||
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
|
||||
classes = n_classes;
|
||||
nBatches = n_batches;
|
||||
confThreshold = conf_thresh;
|
||||
inputCalibs = k_calibs;
|
||||
dim = netRT->input_dim;
|
||||
|
||||
const char *kitti_class_name[] = {
|
||||
"person", "car", "bicycle"};
|
||||
classesNames = std::vector<std::string>(kitti_class_name, std::end( kitti_class_name));
|
||||
|
||||
for(int c=0; c<classes; c++) {
|
||||
int offset = c*123457 % classes;
|
||||
float r = getColor(2, offset, classes);
|
||||
float g = getColor(1, offset, classes);
|
||||
float b = getColor(0, offset, classes);
|
||||
colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
|
||||
}
|
||||
|
||||
src = cv::Mat(cv::Size(2,3), CV_32F);
|
||||
dst = cv::Mat(cv::Size(2,3), CV_32F);
|
||||
dst2 = cv::Mat(cv::Size(2,3), CV_32F);
|
||||
trans = cv::Mat(cv::Size(3,2), CV_32F);
|
||||
trans2 = cv::Mat(cv::Size(3,2), CV_32F);
|
||||
|
||||
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
|
||||
|
||||
dim_hm = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
|
||||
dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
|
||||
dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
|
||||
dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1);
|
||||
dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1);
|
||||
dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
|
||||
|
||||
checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) );
|
||||
checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
|
||||
checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
|
||||
for(int i =0; i<dim_hm.c * dim_hm.h * dim_hm.w; i++){
|
||||
ids_[i] = i;
|
||||
}
|
||||
|
||||
checkCuda( cudaMalloc(&ones, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
|
||||
float *ones_h;
|
||||
checkCuda( cudaMallocHost(&ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
|
||||
for(int i=0; i<dim_dep.c * dim_dep.h * dim_dep.w; i++)
|
||||
ones_h[i]=1.0f;
|
||||
checkCuda( cudaMemcpy(ones, ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float), cudaMemcpyHostToDevice) );
|
||||
checkCuda( cudaFreeHost(ones_h) );
|
||||
|
||||
checkCuda( cudaMallocHost(&scores, K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&scores_d, K *sizeof(float)) );
|
||||
|
||||
checkCuda( cudaMallocHost(&clses, K *sizeof(int)) );
|
||||
checkCuda( cudaMalloc(&clses_d, K *sizeof(int)) );
|
||||
|
||||
checkCuda( cudaMalloc(&topk_inds_d, K *sizeof(int)) );
|
||||
checkCuda( cudaMalloc(&topk_ys_d, K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&topk_xs_d, K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&inttopk_ys_d, K *sizeof(int)) );
|
||||
checkCuda( cudaMalloc(&inttopk_xs_d, K *sizeof(int)) );
|
||||
|
||||
checkCuda( cudaMallocHost(&xs, K * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&ys, K * sizeof(float)) );
|
||||
|
||||
checkCuda( cudaMallocHost(&dep, K * dim_dep.c * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&rot, K * dim_rot.c * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&dim_, K * dim_dim.c * sizeof(float)) );
|
||||
checkCuda( cudaMallocHost(&wh, K * dim_wh.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&dep_d, K * dim_dep.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&rot_d, K * dim_rot.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&dim_d, K * dim_dim.c * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&wh_d, K * dim_wh.c * sizeof(float)) );
|
||||
|
||||
checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) );
|
||||
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
|
||||
checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) );
|
||||
float mean[3] = {0.485, 0.456, 0.406};
|
||||
float stddev[3] = {0.229, 0.224, 0.225};
|
||||
|
||||
checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
|
||||
checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
|
||||
#else
|
||||
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
|
||||
mean << 0.485, 0.456, 0.406;
|
||||
stddev << 0.229, 0.224, 0.225;
|
||||
#endif
|
||||
|
||||
for(int bi=0; bi<nBatches; bi++) {
|
||||
cv::Mat calibs_ = cv::Mat::zeros(cv::Size(4,3), CV_32F);
|
||||
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
|
||||
calibs_.at<float>(0,0) = 707.0493;
|
||||
calibs_.at<float>(0,2) = 604.0814;
|
||||
calibs_.at<float>(1,1) = 707.0493;
|
||||
calibs_.at<float>(1,2) = 180.5066;
|
||||
calibs_.at<float>(0,3) = 45.75831;
|
||||
calibs_.at<float>(1,3) = -0.3454157;
|
||||
calibs_.at<float>(2,2) = 1.0;
|
||||
calibs_.at<float>(2,3) = 0.004981016;
|
||||
}
|
||||
else {
|
||||
calibs_.at<float>(0,0) = inputCalibs[bi].at<float>(0,0);// * (1440.0/dim.w);// / 1440;
|
||||
calibs_.at<float>(0,2) = inputCalibs[bi].at<float>(0,2);// * (1440.0/dim.w);// / 1440;
|
||||
calibs_.at<float>(1,1) = inputCalibs[bi].at<float>(1,1);// * (1080.0/dim.h);//dim.h / 1080;
|
||||
calibs_.at<float>(1,2) = inputCalibs[bi].at<float>(1,2);// * (1080.0/dim.h);//dim.h / 1080;
|
||||
calibs_.at<float>(2,2) = 1.0;
|
||||
}
|
||||
// calibs_.at<float>(0,3) = 45.75831;
|
||||
// calibs_.at<float>(1,3) = -0.3454157;
|
||||
// calibs_.at<float>(2,2) = 1.0;
|
||||
// calibs_.at<float>(2,3) = 0.004981016;
|
||||
calibs.push_back(calibs_);
|
||||
}
|
||||
|
||||
r = cv::Mat(cv::Size(3,3), CV_32F);
|
||||
r.at<float>(0,1) = 0.0;
|
||||
r.at<float>(1,0) = 0.0;
|
||||
r.at<float>(1,1) = 1.0;
|
||||
r.at<float>(1,2) = 0.0;
|
||||
r.at<float>(2,1) = 0.0;
|
||||
|
||||
corners = cv::Mat(cv::Size(8,3), CV_32F);
|
||||
corners.at<float>(1,0) = 0.0;
|
||||
corners.at<float>(1,1) = 0.0;
|
||||
corners.at<float>(1,2) = 0.0;
|
||||
corners.at<float>(1,3) = 0.0;
|
||||
|
||||
pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F);
|
||||
pts3DHomo.at<float>(3,0) = 1.0;
|
||||
pts3DHomo.at<float>(3,1) = 1.0;
|
||||
pts3DHomo.at<float>(3,2) = 1.0;
|
||||
pts3DHomo.at<float>(3,3) = 1.0;
|
||||
pts3DHomo.at<float>(3,4) = 1.0;
|
||||
pts3DHomo.at<float>(3,5) = 1.0;
|
||||
pts3DHomo.at<float>(3,6) = 1.0;
|
||||
pts3DHomo.at<float>(3,7) = 1.0;
|
||||
|
||||
checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) );
|
||||
|
||||
// Alloc array used in the kernel
|
||||
checkCuda( cudaMalloc(&srcOut, K *sizeof(float)) );
|
||||
checkCuda( cudaMalloc(&idsOut, K *sizeof(int)) );
|
||||
|
||||
dst2.at<float>(0,0)=width * 0.5;
|
||||
dst2.at<float>(0,1)=width * 0.5;
|
||||
dst2.at<float>(1,0)=width * 0.5;
|
||||
dst2.at<float>(1,1)=width * 0.5 + width * -0.5;
|
||||
|
||||
dst2.at<float>(2,0)=dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
|
||||
dst2.at<float>(2,1)=dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
|
||||
|
||||
faceId.push_back({0,1,5,4});
|
||||
faceId.push_back({1,2,6, 5});
|
||||
faceId.push_back({2,3,7,6});
|
||||
faceId.push_back({3,0,4,7});
|
||||
// ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]);
|
||||
}
|
||||
|
||||
void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){
|
||||
cv::Size sz = originalSize[bi];
|
||||
float new_height = dim.h;//sz.height * scale;
|
||||
float new_width = dim.w;//sz.width * scale;
|
||||
if(sz.height != sz_old.height && sz.width != sz_old.width){
|
||||
|
||||
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
|
||||
calibs[bi].at<float>(0,2) = new_width / 2.0f;
|
||||
calibs[bi].at<float>(1,2) = new_height /2.0f;
|
||||
}
|
||||
else {
|
||||
calibs[bi].at<float>(0,0) = inputCalibs[bi].at<float>(0,0) * 2.0 * dim.w / sz.width;
|
||||
calibs[bi].at<float>(0,2) = inputCalibs[bi].at<float>(0,2) * dim.w / sz.width ;
|
||||
calibs[bi].at<float>(1,1) = inputCalibs[bi].at<float>(1,1) * 2.0 * dim.h / sz.height;
|
||||
calibs[bi].at<float>(1,2) = inputCalibs[bi].at<float>(1,2) * dim.h / sz.height;
|
||||
}
|
||||
float c[] = {new_width / 2.0f, new_height /2.0f};
|
||||
float s[] = {new_width, new_height};
|
||||
// ----------- get_affine_transform
|
||||
// rot_rad = pi * 0 / 100 --> 0
|
||||
|
||||
src.at<float>(0,0)=c[0];
|
||||
src.at<float>(0,1)=c[1];
|
||||
src.at<float>(1,0)=c[0];
|
||||
src.at<float>(1,1)=c[1] + s[0] * -0.5;
|
||||
dst.at<float>(0,0)=netRT->input_dim.w * 0.5;
|
||||
dst.at<float>(0,1)=netRT->input_dim.h * 0.5;
|
||||
dst.at<float>(1,0)=netRT->input_dim.w * 0.5;
|
||||
dst.at<float>(1,1)=netRT->input_dim.h * 0.5 + netRT->input_dim.w * -0.5;
|
||||
|
||||
src.at<float>(2,0)=src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
|
||||
src.at<float>(2,1)=src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
|
||||
dst.at<float>(2,0)=dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
|
||||
dst.at<float>(2,1)=dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
|
||||
|
||||
trans = cv::getAffineTransform( src, dst );
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME gett affine trans: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
|
||||
trans2 = cv::getAffineTransform( dst2, src );
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
}
|
||||
sz_old = sz;
|
||||
#ifdef OPENCV_CUDACONTRIB
|
||||
// std::cout<<"OPENCV CPMTROB\n";
|
||||
cv::cuda::GpuMat im_Orig;
|
||||
cv::cuda::GpuMat imageF1_d, imageF2_d;
|
||||
|
||||
im_Orig = cv::cuda::GpuMat(frame);
|
||||
cv::cuda::resize (im_Orig, imageF1_d, cv::Size(dim.w, dim.h));//cv::Size(new_width, new_height));
|
||||
// imageF1_d = im_Orig;
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
sz = imageF1_d.size();
|
||||
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME resize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
|
||||
cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR );
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0);
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME convert: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
|
||||
dim2 = dim;
|
||||
cv::cuda::GpuMat bgr[3];
|
||||
cv::cuda::split(imageF1_d,bgr);//split source
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME split: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
|
||||
for(int i=0; i<dim.c; i++)
|
||||
checkCuda( cudaMemcpy(d_ptrs + i*dim.h * dim.w, (float*)bgr[i].data, dim.h * dim.w * sizeof(float), cudaMemcpyDeviceToDevice) );
|
||||
|
||||
normalize(d_ptrs, dim.c, dim.h, dim.w, mean_d, stddev_d);
|
||||
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME normalize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
|
||||
checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
|
||||
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
#else
|
||||
// std::cout<<"NO OPENCV CPMTROB\n";
|
||||
cv::Mat imageF;
|
||||
resize(frame, imageF, cv::Size(dim.w, dim.h));//cv::Size(new_width, new_height));
|
||||
// imageF = frame;
|
||||
sz = imageF.size();
|
||||
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME resize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
|
||||
cv::Mat trans = cv::getAffineTransform( src, dst );
|
||||
cv::warpAffine(imageF, imageF, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR );
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME warpAffine: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
|
||||
sz = imageF.size();
|
||||
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
|
||||
imageF.convertTo(imageF, CV_32FC3, 1/255.0);
|
||||
// end_t = std::chrono::steady_clock::now();
|
||||
// std::cout << " TIME convertto: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
|
||||
// step_t = end_t;
|
||||
dim2 = dim;
|
||||
//split channels
|
||||
cv::Mat bgr[3];
|
||||
cv::split(imageF,bgr);//split source
|
||||
for(int i=0; i<3; i++){
|
||||
bgr[i] = bgr[i] - mean[i];
|
||||
bgr[i] = bgr[i] / stddev[i];
|
||||
}
|
||||
|
||||
//write channels
|
||||
for(int i=0; i<dim2.c; i++) {
|
||||
int idx = i*imageF.rows*imageF.cols;
|
||||
int ch = dim2.c-3 +i;
|
||||
// std::cout<<"i: "<<i<<", idx: "<<idx<<", ch: "<<ch<<std::endl;
|
||||
memcpy((void*)&input[idx+ netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
|
||||
}
|
||||
checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
|
||||
#endif
|
||||
}
|
||||
|
||||
void CenternetDetection3D::postprocess(const int bi, const bool mAP) {
|
||||
dnnType *rt_out[7];
|
||||
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
|
||||
rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
|
||||
rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
|
||||
rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
|
||||
rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi;
|
||||
rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi;
|
||||
rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi;
|
||||
|
||||
// ------------------------------------ process --------------------------------------------
|
||||
activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot());
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
// output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1.
|
||||
activationSIGMOIDForward(rt_out[4], rt_out[4], dim_dep.tot());
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
transformDep(ones, ones + dim_dep.tot(), rt_out[4], rt_out[4] + dim_dep.tot());
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op);
|
||||
|
||||
// ----------- nms end
|
||||
// ----------- topk
|
||||
|
||||
if(K > dim_hm.h * dim_hm.w){
|
||||
printf ("Error topk (K is too large)\n");
|
||||
return;
|
||||
}
|
||||
|
||||
checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );
|
||||
|
||||
sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d);
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d);
|
||||
checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d);
|
||||
|
||||
checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
|
||||
checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
|
||||
|
||||
checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
|
||||
|
||||
// ----------- topk end
|
||||
|
||||
topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], srcOut, idsOut);
|
||||
// checkCuda( cudaDeviceSynchronize() );
|
||||
|
||||
|
||||
getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[4], dep_d, idsOut);
|
||||
checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[5], rot_d, idsOut);
|
||||
checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[6], dim_d, idsOut);
|
||||
checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
getRecordsFromTopKId(topk_inds_d, K, dim_wh.c, dim_wh.h * dim_wh.w, rt_out[2], wh_d, idsOut);
|
||||
checkCuda( cudaMemcpy(wh, wh_d, K * dim_wh.c * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
checkCuda( cudaMemcpy(xs, topk_xs_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
checkCuda( cudaMemcpy(ys, topk_ys_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
|
||||
|
||||
// ---------------------------------- post-process -----------------------------------------
|
||||
|
||||
// ddd_post_process_2d
|
||||
cv::Mat new_pt1(cv::Size(1,2), CV_32F);
|
||||
cv::Mat new_pt2(cv::Size(1,2), CV_32F);
|
||||
|
||||
for(int i = 0; i<K; i++){
|
||||
new_pt1.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*xs[i] +
|
||||
static_cast<float>(trans2.at<double>(0,1))*ys[i] +
|
||||
static_cast<float>(trans2.at<double>(0,2))*1.0;
|
||||
new_pt1.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*xs[i] +
|
||||
static_cast<float>(trans2.at<double>(1,1))*ys[i] +
|
||||
static_cast<float>(trans2.at<double>(1,2))*1.0;
|
||||
|
||||
new_pt2.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*wh[i] +
|
||||
static_cast<float>(trans2.at<double>(0,1))*wh[K+i] +
|
||||
static_cast<float>(trans2.at<double>(0,2))*1.0;
|
||||
new_pt2.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*wh[i] +
|
||||
static_cast<float>(trans2.at<double>(1,1))*wh[K+i] +
|
||||
static_cast<float>(trans2.at<double>(1,2))*1.0;
|
||||
|
||||
target_coords[i*4] = new_pt1.at<float>(0,0);
|
||||
target_coords[i*4+1] = new_pt1.at<float>(0,1);
|
||||
target_coords[i*4+2] = new_pt2.at<float>(0,0);
|
||||
target_coords[i*4+3] = new_pt2.at<float>(0,1);
|
||||
}
|
||||
|
||||
float alpha;
|
||||
float x, y, z, rot_y;
|
||||
detected3D.clear();
|
||||
for(int i = 0; i<classes; i++){
|
||||
for(int j=0; j<K; j++){
|
||||
if(clses[j] == i){
|
||||
//get alpha
|
||||
if(rot[1*K + j] > rot[5*K + j])
|
||||
alpha = std::atan2(rot[2*K + j], rot[3*K + j]) -0.5 * M_PI;
|
||||
else
|
||||
alpha = std::atan2(rot[6*K + j], rot[7*K + j]) +0.5 * M_PI;
|
||||
|
||||
// unproject_2d_to_3d
|
||||
z = dep[j] - calibs[bi].at<float>(2,3);// z = depth - P[2, 3]
|
||||
x = (target_coords[j*4] * dep[j] - calibs[bi].at<float>(0,3) - calibs[bi].at<float>(0,2) * z) / calibs[bi].at<float>(0,0);
|
||||
y = (target_coords[j*4+1] * dep[j] - calibs[bi].at<float>(1,3) - calibs[bi].at<float>(1,2) * z) / calibs[bi].at<float>(1,1) + (dim_[j] / 2);
|
||||
// alpha2rot_y
|
||||
rot_y = (alpha + std::atan2(target_coords[j*4] - calibs[bi].at<float>(0,2), calibs[bi].at<float>(0,0)));
|
||||
if(rot_y>M_PI)
|
||||
rot_y -= 2*M_PI;
|
||||
if(rot_y<M_PI)
|
||||
rot_y += 2*M_PI;
|
||||
|
||||
if(scores[j] > confThreshold) {
|
||||
if(z>0) {
|
||||
// compute_box_3d
|
||||
r.at<float>(0,0) = std::cos(rot_y);
|
||||
r.at<float>(0,2) = std::sin(rot_y);
|
||||
r.at<float>(2,0) = -std::sin(rot_y);
|
||||
r.at<float>(2,2) = std::cos(rot_y);
|
||||
|
||||
corners.at<float>(0,0) = dim_[2*K+j]/2;
|
||||
corners.at<float>(0,1) = dim_[2*K+j]/2;
|
||||
corners.at<float>(0,2) = -dim_[2*K+j]/2;
|
||||
corners.at<float>(0,3) = -dim_[2*K+j]/2;
|
||||
corners.at<float>(0,4) = dim_[2*K+j]/2;
|
||||
corners.at<float>(0,5) = dim_[2*K+j]/2;
|
||||
corners.at<float>(0,6) = -dim_[2*K+j]/2;
|
||||
corners.at<float>(0,7) = -dim_[2*K+j]/2;
|
||||
|
||||
corners.at<float>(1,4) = -dim_[j];
|
||||
corners.at<float>(1,5) = -dim_[j];
|
||||
corners.at<float>(1,6) = -dim_[j];
|
||||
corners.at<float>(1,7) = -dim_[j];
|
||||
|
||||
corners.at<float>(2,0) = dim_[K+j]/2;
|
||||
corners.at<float>(2,1) = -dim_[K+j]/2;
|
||||
corners.at<float>(2,2) = -dim_[K+j]/2;
|
||||
corners.at<float>(2,3) = dim_[K+j]/2;
|
||||
corners.at<float>(2,4) = dim_[K+j]/2;
|
||||
corners.at<float>(2,5) = -dim_[K+j]/2;
|
||||
corners.at<float>(2,6) = -dim_[K+j]/2;
|
||||
corners.at<float>(2,7) = dim_[K+j]/2;
|
||||
cv::Mat aus = r * corners;
|
||||
|
||||
for(int k=0; k<8; k++) {
|
||||
aus.at<float>(0,k) += x;
|
||||
aus.at<float>(1,k) += y;
|
||||
aus.at<float>(2,k) += z;
|
||||
}
|
||||
// corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3)));
|
||||
for(int k1=0; k1<3; k1++) {
|
||||
for(int k2=0; k2<8; k2++)
|
||||
pts3DHomo.at<float>(k1,k2) = aus.at<float>(k1,k2);
|
||||
}
|
||||
aus.release();
|
||||
aus = calibs[bi] * pts3DHomo;
|
||||
|
||||
tk::dnn::box3D res;
|
||||
for(int k=0; k<8; k++) {
|
||||
res.corners.push_back(aus.at<float>(0,k) / aus.at<float>(2,k));
|
||||
res.corners.push_back(aus.at<float>(1,k) / aus.at<float>(2,k));
|
||||
}
|
||||
res.cl = i;
|
||||
res.prob = scores[j];
|
||||
//res.print();
|
||||
detected3D.push_back(res);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
batchDetected.push_back(detected3D);
|
||||
}
|
||||
|
||||
void CenternetDetection3D::draw(std::vector<cv::Mat>& frames) {
|
||||
tk::dnn::box3D b;
|
||||
int x0, w, x1, y0, h, y1;
|
||||
int objClass;
|
||||
std::string det_class;
|
||||
|
||||
int baseline = 0;
|
||||
float font_scale = 0.5;
|
||||
int thickness = 2;
|
||||
|
||||
for(int bi=0; bi<frames.size(); ++bi){
|
||||
float scale_x = float(originalSize[bi].width)/dim.w;
|
||||
float scale_y = float(originalSize[bi].height)/dim.h;
|
||||
resize(frames[bi], frames[bi], originalSize[bi]);
|
||||
// draw dets
|
||||
for(int i=0; i<batchDetected[bi].size(); i++) {
|
||||
b = batchDetected[bi][i];
|
||||
|
||||
for(int ind_f = 3; ind_f>=0; ind_f--) {
|
||||
for(int j=0; j<4; j++) {
|
||||
cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(j) * 2) * scale_x,
|
||||
b.corners.at(faceId.at(ind_f).at(j) * 2 + 1) * scale_y),
|
||||
cv::Point(b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2) * scale_x,
|
||||
b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1) * scale_y),
|
||||
colors[b.cl], 2);
|
||||
if(ind_f == 0) {
|
||||
cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(0) * 2) * scale_x,
|
||||
b.corners.at(faceId.at(ind_f).at(0) * 2 + 1)* scale_y),
|
||||
cv::Point(b.corners.at(faceId.at(ind_f).at(2) * 2) * scale_x,
|
||||
b.corners.at(faceId.at(ind_f).at(2) * 2 + 1) * scale_y), colors[b.cl], 2);
|
||||
cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(1) * 2)* scale_x,
|
||||
b.corners.at(faceId.at(ind_f).at(1) * 2 + 1)* scale_y),
|
||||
cv::Point(b.corners.at(faceId.at(ind_f).at(3) * 2)* scale_x,
|
||||
b.corners.at(faceId.at(ind_f).at(3) * 2 + 1)* scale_y), colors[b.cl], 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
// draw label
|
||||
cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
|
||||
cv::rectangle(frames[bi], cv::Point(b.corners.at(faceId.at(0).at(0) * 2)* scale_x,
|
||||
b.corners.at(faceId.at(0).at(0) * 2 + 1)* scale_y),
|
||||
cv::Point((b.corners.at(faceId.at(0).at(0) * 2)* scale_x + text_size.width - 2),
|
||||
(b.corners.at(faceId.at(0).at(0) * 2 + 1)* scale_y - text_size.height - 2)), colors[b.cl], -1);
|
||||
cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(faceId.at(0).at(0) * 2)* scale_x,
|
||||
(b.corners.at(faceId.at(0).at(0) * 2 + 1)* scale_y - (baseline / 2))),
|
||||
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}}
|
||||
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
|
||||
#include "demo_utils.h"
|
||||
|
||||
void readCalibrationMatrix(const std::string& path, cv::Mat& calib_mat){
|
||||
YAML::Node config = YAML::LoadFile(path);
|
||||
|
||||
//read camera matrix
|
||||
int rows = config["camera_matrix"]["rows"].as<int>();
|
||||
int cols = config["camera_matrix"]["cols"].as<int>();
|
||||
|
||||
cv::Mat calib = cv::Mat(cv::Size(rows, cols), CV_32F);
|
||||
float *vals = (float *)calib.data;
|
||||
|
||||
for(int i=0; i < config["camera_matrix"]["data"].size(); ++i )
|
||||
vals[i] = config["camera_matrix"]["data"][i].as<float>();
|
||||
|
||||
calib_mat = calib;
|
||||
|
||||
}
|
||||
@@ -1,5 +1,11 @@
|
||||
#include "kernelsThrust.h"
|
||||
|
||||
void transformDep(float *src_begin, float *src_end, float *dst_begin, float *dst_end) {
|
||||
int e = exp(-6);
|
||||
thrust::transform(thrust::device, dst_begin, dst_end, thrust::make_constant_iterator(e), dst_begin, thrust::plus<float>());
|
||||
thrust::transform(thrust::device, src_begin, src_end, dst_begin, dst_begin, thrust::divides<float>());
|
||||
thrust::transform(thrust::device, dst_begin, dst_end, thrust::make_constant_iterator(-1.0), dst_begin, thrust::plus<float>());
|
||||
}
|
||||
|
||||
void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op){
|
||||
thrust::transform(thrust::device, src_begin, src_end, src2_begin, src_out, op);
|
||||
@@ -70,6 +76,14 @@ void topKxyAddOffset(int * ids_begin, const int K, const int size,
|
||||
thrust::transform(thrust::device, intys_begin, intys_begin + K, src_out, ys_begin, thrust::plus<float>());
|
||||
}
|
||||
|
||||
void getRecordsFromTopKId(int * ids_begin, const int K, const int ch, const int size, dnnType *src_begin, float *src_out, int *ids_out) {
|
||||
for(int i=0; i<ch; i++) {
|
||||
// thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out);
|
||||
thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(i*size), ids_out, thrust::plus<int>());
|
||||
thrust::gather(thrust::device, ids_out, ids_out + K, src_begin, src_out+i*K);
|
||||
}
|
||||
}
|
||||
|
||||
void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin,
|
||||
dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1,
|
||||
float *src_out, int *ids_out){
|
||||
|
||||
+1
-1
@@ -243,4 +243,4 @@ void removePathAndExtension(const std::string &full_string, std::string &name){
|
||||
name = name.substr(previous, current);
|
||||
|
||||
// std::cout<<"full string: "<<full_string<<" name: "<<name<<std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,562 @@
|
||||
#include <iostream>
|
||||
#include "tkdnn.h"
|
||||
|
||||
const char *input_bin = "dla34_cnet3d/debug/input.bin";
|
||||
const char *conv1_bin = "dla34_cnet3d/layers/base-base_layer-0.bin";
|
||||
const char *conv2_bin = "dla34_cnet3d/layers/base-level0-0.bin";
|
||||
const char *conv3_bin = "dla34_cnet3d/layers/base-level1-0.bin";
|
||||
// s - stage, t - tree
|
||||
const char *s1_t1_conv1_bin = "dla34_cnet3d/layers/base-level2-tree1-conv1.bin";
|
||||
const char *s1_t1_conv2_bin = "dla34_cnet3d/layers/base-level2-tree1-conv2.bin";
|
||||
const char *s1_t1_project = "dla34_cnet3d/layers/base-level2-project-0.bin";
|
||||
const char *s1_t2_conv1_bin = "dla34_cnet3d/layers/base-level2-tree2-conv1.bin";
|
||||
const char *s1_t2_conv2_bin = "dla34_cnet3d/layers/base-level2-tree2-conv2.bin";
|
||||
const char *s1_root_conv1_bin = "dla34_cnet3d/layers/base-level2-root-conv.bin";
|
||||
const char *s2_t1_t1_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-tree1-conv1.bin";
|
||||
const char *s2_t1_t1_conv2_bin = "dla34_cnet3d/layers/base-level3-tree1-tree1-conv2.bin";
|
||||
const char *s2_t1_t1_project = "dla34_cnet3d/layers/base-level3-tree1-project-0.bin";
|
||||
const char *s2_t1_t2_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-tree2-conv1.bin";
|
||||
const char *s2_t1_t2_conv2_bin = "dla34_cnet3d/layers/base-level3-tree1-tree2-conv2.bin";
|
||||
const char *s2_t1_root_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-root-conv.bin";
|
||||
const char *s2_t2_t1_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-tree1-conv1.bin";
|
||||
const char *s2_t2_t1_conv2_bin = "dla34_cnet3d/layers/base-level3-tree2-tree1-conv2.bin";
|
||||
const char *s2_t2_t2_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-tree2-conv1.bin";
|
||||
const char *s2_t2_t2_conv2_bin = "dla34_cnet3d/layers/base-level3-tree2-tree2-conv2.bin";
|
||||
const char *s2_t2_root_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-root-conv.bin";
|
||||
const char *s3_t1_t1_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-tree1-conv1.bin";
|
||||
const char *s3_t1_t1_conv2_bin = "dla34_cnet3d/layers/base-level4-tree1-tree1-conv2.bin";
|
||||
const char *s3_t1_t1_project = "dla34_cnet3d/layers/base-level4-tree1-project-0.bin";
|
||||
const char *s3_t1_t2_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-tree2-conv1.bin";
|
||||
const char *s3_t1_t2_conv2_bin = "dla34_cnet3d/layers/base-level4-tree1-tree2-conv2.bin";
|
||||
const char *s3_t1_root_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-root-conv.bin";
|
||||
const char *s3_t2_t1_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-tree1-conv1.bin";
|
||||
const char *s3_t2_t1_conv2_bin = "dla34_cnet3d/layers/base-level4-tree2-tree1-conv2.bin";
|
||||
const char *s3_t2_t2_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-tree2-conv1.bin";
|
||||
const char *s3_t2_t2_conv2_bin = "dla34_cnet3d/layers/base-level4-tree2-tree2-conv2.bin";
|
||||
const char *s3_t2_root_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-root-conv.bin";
|
||||
const char *s4_t1_conv1_bin = "dla34_cnet3d/layers/base-level5-tree1-conv1.bin";
|
||||
const char *s4_t1_conv2_bin = "dla34_cnet3d/layers/base-level5-tree1-conv2.bin";
|
||||
const char *s4_t1_project = "dla34_cnet3d/layers/base-level5-project-0.bin";
|
||||
const char *s4_t2_conv1_bin = "dla34_cnet3d/layers/base-level5-tree2-conv1.bin";
|
||||
const char *s4_t2_conv2_bin = "dla34_cnet3d/layers/base-level5-tree2-conv2.bin";
|
||||
const char *s4_root_conv1_bin = "dla34_cnet3d/layers/base-level5-root-conv.bin";
|
||||
|
||||
//final
|
||||
// const char *fc_bin = "dla34_cnet3d/layers/output.bin";
|
||||
|
||||
const char *ida_0_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_0-proj_1-conv.bin";
|
||||
const char *ida_0_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_0_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_0-up_1.bin";
|
||||
const char *ida_0_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_0-node_1-conv.bin";
|
||||
const char *ida_0_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin";
|
||||
|
||||
const char *ida_1_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_1-conv.bin";
|
||||
const char *ida_1_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_1_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_1-up_1.bin";
|
||||
const char *ida_1_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_1-conv.bin";
|
||||
const char *ida_1_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_1_p_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_2-conv.bin";
|
||||
const char *ida_1_p_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin";
|
||||
const char *ida_1_up_2_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_1-up_2.bin";
|
||||
const char *ida_1_n_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_2-conv.bin";
|
||||
const char *ida_1_n_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin";
|
||||
|
||||
const char *ida_2_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_1-conv.bin";
|
||||
const char *ida_2_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_1.bin";
|
||||
const char *ida_2_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_1-conv.bin";
|
||||
const char *ida_2_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_p_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_2-conv.bin";
|
||||
const char *ida_2_p_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_up_2_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_2.bin";
|
||||
const char *ida_2_n_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_2-conv.bin";
|
||||
const char *ida_2_n_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_p_3_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_3-conv.bin";
|
||||
const char *ida_2_p_3_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_up_3_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_3.bin";
|
||||
const char *ida_2_n_3_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_3-conv.bin";
|
||||
const char *ida_2_n_3_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin";
|
||||
|
||||
const char *ida_up_p_1_dcn_bin = "dla34_cnet3d/layers/ida_up-proj_1-conv.bin";
|
||||
const char *ida_up_p_1_conv_bin = "dla34_cnet3d/layers/ida_up-proj_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_up_up_1_deconv_bin = "dla34_cnet3d/layers/ida_up-up_1.bin";
|
||||
const char *ida_up_n_1_dcn_bin = "dla34_cnet3d/layers/ida_up-node_1-conv.bin";
|
||||
const char *ida_up_n_1_conv_bin = "dla34_cnet3d/layers/ida_up-node_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_up_p_2_dcn_bin = "dla34_cnet3d/layers/ida_up-proj_2-conv.bin";
|
||||
const char *ida_up_p_2_conv_bin = "dla34_cnet3d/layers/ida_up-proj_2-conv-conv_offset_mask.bin";
|
||||
const char *ida_up_up_2_deconv_bin = "dla34_cnet3d/layers/ida_up-up_2.bin";
|
||||
const char *ida_up_n_2_dcn_bin = "dla34_cnet3d/layers/ida_up-node_2-conv.bin";
|
||||
const char *ida_up_n_2_conv_bin = "dla34_cnet3d/layers/ida_up-node_2-conv-conv_offset_mask.bin";
|
||||
|
||||
const char *hm_conv1_bin = "dla34_cnet3d/layers/hm-0.bin";
|
||||
const char *hm_conv2_bin = "dla34_cnet3d/layers/hm-2.bin";
|
||||
const char *wh_conv1_bin = "dla34_cnet3d/layers/wh-0.bin";
|
||||
const char *wh_conv2_bin = "dla34_cnet3d/layers/wh-2.bin";
|
||||
const char *reg_conv1_bin = "dla34_cnet3d/layers/reg-0.bin";
|
||||
const char *reg_conv2_bin = "dla34_cnet3d/layers/reg-2.bin";
|
||||
const char *dep_conv1_bin = "dla34_cnet3d/layers/dep-0.bin";
|
||||
const char *dep_conv2_bin = "dla34_cnet3d/layers/dep-2.bin";
|
||||
const char *rot_conv1_bin = "dla34_cnet3d/layers/rot-0.bin";
|
||||
const char *rot_conv2_bin = "dla34_cnet3d/layers/rot-2.bin";
|
||||
const char *dim_conv1_bin = "dla34_cnet3d/layers/dim-0.bin";
|
||||
const char *dim_conv2_bin = "dla34_cnet3d/layers/dim-2.bin";
|
||||
|
||||
const char *output_bin[]={
|
||||
"dla34_cnet3d/debug/hm.bin",
|
||||
"dla34_cnet3d/debug/wh.bin",
|
||||
"dla34_cnet3d/debug/reg.bin",
|
||||
"dla34_cnet3d/debug/dep.bin",
|
||||
"dla34_cnet3d/debug/rot.bin",
|
||||
"dla34_cnet3d/debug/dim.bin"};
|
||||
|
||||
int main()
|
||||
{
|
||||
|
||||
downloadWeightsifDoNotExist(input_bin, "dla34_cnet3d", "https://cloud.hipert.unimore.it/s/2MDyWGzQsTKMjmR/download");
|
||||
|
||||
// Network layout
|
||||
tk::dnn::dataDim_t dim(1, 3, 512, 512, 1);
|
||||
tk::dnn::Network net(dim);
|
||||
tk::dnn::Layer *last1, *last2, *last3, *last4;
|
||||
tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2;
|
||||
|
||||
tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true);
|
||||
tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true);
|
||||
tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU);
|
||||
base1 = &relu2;
|
||||
|
||||
tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true);
|
||||
tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU);
|
||||
base2 = &relu3;
|
||||
|
||||
// level 2
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true);
|
||||
last2 = &s1_t1_conv2;
|
||||
|
||||
// get the basicblock input and apply maxpool conv2d and relu
|
||||
tk::dnn::Layer *route_s1_t1_layers[1] = { base2 };
|
||||
tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1);
|
||||
// downsample
|
||||
tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
|
||||
// project
|
||||
tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true);
|
||||
|
||||
tk::dnn::Shortcut s1_t1_s1(&net, last2);
|
||||
tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s1_t1_relu;
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s1_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s1_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 128, 56, 56
|
||||
tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 };
|
||||
tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2);
|
||||
tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true);
|
||||
tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
base3 = &s1_root_relu;
|
||||
|
||||
// level 3
|
||||
// tree 1
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true);
|
||||
last2 = &s2_t1_t1_conv2;
|
||||
|
||||
// get the basicblock input and apply maxpool conv2d and relu
|
||||
tk::dnn::Layer *route_s2_t1_t1_layers[1] = { base3 };
|
||||
tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1);
|
||||
// downsample
|
||||
tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
|
||||
last4 = &s2_t1_t1_maxpool1;
|
||||
// project
|
||||
tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true);
|
||||
|
||||
tk::dnn::Shortcut s2_t1_t1_s1(&net, last2);
|
||||
tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s2_t1_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s2_t1_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s2_t1_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 128, 56, 56
|
||||
tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 };
|
||||
tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2);
|
||||
tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s2_t1_root_relu;
|
||||
last3 = &s2_t1_root_relu;
|
||||
// tree 2
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true);
|
||||
tk::dnn::Shortcut s2_t2_t1_s1(&net, last1);
|
||||
tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s2_t2_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s2_t2_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s2_t2_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 128, 56, 56
|
||||
tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3};
|
||||
tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4);
|
||||
tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
base4 = &s2_t2_root_relu;
|
||||
|
||||
// level 4
|
||||
// tree 1
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true);
|
||||
last2 = &s3_t1_t1_conv2;
|
||||
|
||||
// get the basicblock input and apply maxpool conv2d and relu
|
||||
tk::dnn::Layer *route_s3_t1_t1_layers[1] = { base4 };
|
||||
tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1);
|
||||
// downsample
|
||||
tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
|
||||
last4 = &s3_t1_t1_maxpool1;
|
||||
// project
|
||||
tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true);
|
||||
|
||||
tk::dnn::Shortcut s3_t1_t1_s1(&net, last2);
|
||||
tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s3_t1_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s3_t1_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s3_t1_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 256, 56, 56
|
||||
tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 };
|
||||
tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2);
|
||||
tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s3_t1_root_relu;
|
||||
last3 = &s3_t1_root_relu;
|
||||
// tree 2
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true);
|
||||
tk::dnn::Shortcut s3_t2_t1_s1(&net, last1);
|
||||
tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s3_t2_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s3_t2_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s3_t2_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 256, 56, 56
|
||||
tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3};
|
||||
tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4);
|
||||
tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
base5 = &s3_t2_root_relu;
|
||||
|
||||
// level 5
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true);
|
||||
last2 = &s4_t1_conv2;
|
||||
|
||||
// get the basicblock input and apply maxpool conv2d and relu
|
||||
tk::dnn::Layer *route_s4_t1_layers[1] = { base5 };
|
||||
tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1);
|
||||
// downsample
|
||||
tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
|
||||
last4 = &s4_t1_maxpool1;
|
||||
// project
|
||||
tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true);
|
||||
|
||||
tk::dnn::Shortcut s4_t1_s1(&net, last2);
|
||||
tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s4_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s4_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s4_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 128, 56, 56
|
||||
tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 };
|
||||
tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3);
|
||||
tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true);
|
||||
tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
base6 = &s4_root_relu;
|
||||
|
||||
//final
|
||||
// tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE);
|
||||
// tk::dnn::Dense fc(&net, 1000, fc_bin);
|
||||
|
||||
//ida 0
|
||||
tk::dnn::DeformConv2d ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256);
|
||||
tk::dnn::Shortcut ida_0_shortcut(&net, base5);
|
||||
tk::dnn::DeformConv2d ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida1 = &ida_0_n_1_relu;
|
||||
|
||||
//ida1-1
|
||||
tk::dnn::Layer *route_ida1_layers_1[1] = { base5 };
|
||||
tk::dnn::Route route_ida1_1(&net, route_ida1_layers_1, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128);
|
||||
tk::dnn::Shortcut ida_1_shortcut1(&net, base4);
|
||||
tk::dnn::DeformConv2d ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida2_1 = &ida_1_n_1_relu;
|
||||
|
||||
//ida1-2
|
||||
tk::dnn::Layer *route_ida1_layers_2[1] = { ida1 };
|
||||
tk::dnn::Route route_ida1_2(&net, route_ida1_layers_2, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true);
|
||||
tk::dnn::Activation ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128);
|
||||
tk::dnn::Shortcut ida_1_shortcut2(&net, ida2_1);
|
||||
tk::dnn::DeformConv2d ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true);
|
||||
tk::dnn::Activation ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida2_2 = &ida_1_n_2_relu;
|
||||
|
||||
//ida2-1
|
||||
tk::dnn::Layer *route_ida2_layers_1[1] = { base4 };
|
||||
tk::dnn::Route route_ida2_1(&net, route_ida2_layers_1, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut ida_2_shortcut1(&net, base3);
|
||||
tk::dnn::DeformConv2d ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida3_1 = &ida_2_n_1_relu;
|
||||
|
||||
//ida2-2
|
||||
tk::dnn::Layer *route_ida2_layers_2[1] = { ida2_1 };
|
||||
tk::dnn::Route route_ida2_2(&net, route_ida2_layers_2, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut ida_2_shortcut2(&net, ida3_1);
|
||||
tk::dnn::DeformConv2d ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida3_2 = &ida_2_n_2_relu;
|
||||
|
||||
//ida2-3
|
||||
tk::dnn::Layer *route_ida2_layers_3[1] = { ida2_2 };
|
||||
tk::dnn::Route route_ida2_3(&net, route_ida2_layers_3, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut ida_2_shortcut3(&net, ida3_2);
|
||||
tk::dnn::DeformConv2d ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida3_3 = &ida_2_n_3_relu;
|
||||
|
||||
//idaup-1
|
||||
tk::dnn::Layer *route_idaup_layers_1[1] = { ida2_2 };
|
||||
tk::dnn::Route route_idaup_1(&net, route_idaup_layers_1, 1);
|
||||
|
||||
tk::dnn::DeformConv2d idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true);
|
||||
tk::dnn::Activation idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut idaup_shortcut1(&net, ida3_3);
|
||||
tk::dnn::DeformConv2d idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true);
|
||||
tk::dnn::Activation idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
idaup_1 = &idaup_n_1_relu;
|
||||
|
||||
//idaup-2
|
||||
tk::dnn::Layer *route_idaup_layers_2[1] = { ida1 };
|
||||
tk::dnn::Route route_idaup_2(&net, route_idaup_layers_2, 1);
|
||||
|
||||
tk::dnn::DeformConv2d idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true);
|
||||
tk::dnn::Activation idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut idaup_shortcut2(&net, idaup_1);
|
||||
tk::dnn::DeformConv2d idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true);
|
||||
tk::dnn::Activation idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
idaup_2 = &idaup_n_2_relu;
|
||||
|
||||
tk::dnn::Layer *route_1_0_layers[1] = { idaup_2 };
|
||||
|
||||
// hm
|
||||
tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false);
|
||||
tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false);
|
||||
hm->setFinal();
|
||||
int kernel = 3;
|
||||
int pad = (kernel - 1)/2;
|
||||
tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID);
|
||||
tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX);
|
||||
hmax->setFinal();
|
||||
|
||||
// wh
|
||||
tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false);
|
||||
tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false);
|
||||
wh->setFinal();
|
||||
|
||||
// reg
|
||||
tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false);
|
||||
tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false);
|
||||
reg->setFinal();
|
||||
|
||||
// dep
|
||||
tk::dnn::Route *route_3_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *dep_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dep_conv1_bin, false);
|
||||
tk::dnn::Activation *dep_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *dep = new tk::dnn::Conv2d(&net, 1, 1, 1, 1, 1, 0, 0, dep_conv2_bin, false);
|
||||
dep->setFinal();
|
||||
|
||||
// rot
|
||||
tk::dnn::Route *route_4_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *rot_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, rot_conv1_bin, false);
|
||||
tk::dnn::Activation *rot_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *rot = new tk::dnn::Conv2d(&net, 8, 1, 1, 1, 1, 0, 0, rot_conv2_bin, false);
|
||||
rot->setFinal();
|
||||
|
||||
// dim
|
||||
tk::dnn::Route *route_5_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *dim_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dim_conv1_bin, false);
|
||||
tk::dnn::Activation *dim_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *dim_ = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, dim_conv2_bin, false);
|
||||
dim_->setFinal();
|
||||
|
||||
// Load input
|
||||
dnnType *data;
|
||||
dnnType *input_h;
|
||||
readBinaryFile(input_bin, dim.tot(), &input_h, &data);
|
||||
//printDeviceVector(64, data, true);
|
||||
|
||||
//print network model
|
||||
net.print();
|
||||
|
||||
//convert network to tensorRT
|
||||
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet3d"));
|
||||
|
||||
tk::dnn::dataDim_t dim1 = dim; //input dim
|
||||
printCenteredTitle(" CUDNN inference ", '=', 30);
|
||||
{
|
||||
dim1.print();
|
||||
TKDNN_TSTART
|
||||
net.infer(dim1, data);
|
||||
TKDNN_TSTOP
|
||||
dim1.print();
|
||||
}
|
||||
|
||||
tk::dnn::dataDim_t dim2 = dim;
|
||||
printCenteredTitle(" TENSORRT inference ", '=', 30);
|
||||
{
|
||||
dim2.print();
|
||||
TKDNN_TSTART
|
||||
netRT.infer(dim2, data);
|
||||
TKDNN_TSTOP
|
||||
dim2.print();
|
||||
}
|
||||
|
||||
tk::dnn::Layer *outs[6] = { hm, wh, reg, dep, rot, dim_ };
|
||||
int out_count = 1;
|
||||
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
|
||||
for(int i=0; i<6; i++) {
|
||||
printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30);
|
||||
|
||||
outs[i]->output_dim.print();
|
||||
|
||||
dnnType *out, *out_h;
|
||||
int odim = outs[i]->output_dim.tot();
|
||||
readBinaryFile(output_bin[i], odim, &out_h, &out);
|
||||
|
||||
dnnType *cudnn_out, *rt_out;
|
||||
cudnn_out = outs[i]->dstData;
|
||||
rt_out = (dnnType *)netRT.buffersRT[i+out_count];
|
||||
// there is the maxpool. It isn't an output but it is necessary for the process section
|
||||
if(i==0)
|
||||
out_count ++;
|
||||
|
||||
std::cout<<"CUDNN vs correct";
|
||||
ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN;
|
||||
std::cout<<"TRT vs correct";
|
||||
ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT;
|
||||
std::cout<<"CUDNN vs TRT ";
|
||||
ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
|
||||
}
|
||||
return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
|
||||
}
|
||||
@@ -0,0 +1,632 @@
|
||||
#include <iostream>
|
||||
#include "tkdnn.h"
|
||||
const char *input_bin = "dla34_ctrack/debug/input_base-level0-0.bin";
|
||||
// const char *input_bin = "dla34_ctrack/debug/input.bin";
|
||||
// const char *pre_img_bin = "dla34_ctrack/debug/pre_imgages.bin";
|
||||
// const char *pre_hm_bin = "dla34_ctrack/debug/pre_hms.bin";
|
||||
// //pre
|
||||
// const char *pre_img_conv1_bin = "dla34_ctrack/layers/base-pre_img_layer-0.bin";
|
||||
// const char *pre_hm_conv1_bin = "dla34_ctrack/layers/base-pre_hm_layer-0.bin";
|
||||
// const char *conv1_bin = "dla34_ctrack/layers/base-base_layer-0.bin";
|
||||
|
||||
const char *conv2_bin = "dla34_ctrack/layers/base-level0-0.bin";
|
||||
const char *conv3_bin = "dla34_ctrack/layers/base-level1-0.bin";
|
||||
// s - stage, t - tree
|
||||
const char *s1_t1_conv1_bin = "dla34_ctrack/layers/base-level2-tree1-conv1.bin";
|
||||
const char *s1_t1_conv2_bin = "dla34_ctrack/layers/base-level2-tree1-conv2.bin";
|
||||
const char *s1_t1_project = "dla34_ctrack/layers/base-level2-project-0.bin";
|
||||
const char *s1_t2_conv1_bin = "dla34_ctrack/layers/base-level2-tree2-conv1.bin";
|
||||
const char *s1_t2_conv2_bin = "dla34_ctrack/layers/base-level2-tree2-conv2.bin";
|
||||
const char *s1_root_conv1_bin = "dla34_ctrack/layers/base-level2-root-conv.bin";
|
||||
const char *s2_t1_t1_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-tree1-conv1.bin";
|
||||
const char *s2_t1_t1_conv2_bin = "dla34_ctrack/layers/base-level3-tree1-tree1-conv2.bin";
|
||||
const char *s2_t1_t1_project = "dla34_ctrack/layers/base-level3-tree1-project-0.bin";
|
||||
const char *s2_t1_t2_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-tree2-conv1.bin";
|
||||
const char *s2_t1_t2_conv2_bin = "dla34_ctrack/layers/base-level3-tree1-tree2-conv2.bin";
|
||||
const char *s2_t1_root_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-root-conv.bin";
|
||||
const char *s2_t2_t1_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-tree1-conv1.bin";
|
||||
const char *s2_t2_t1_conv2_bin = "dla34_ctrack/layers/base-level3-tree2-tree1-conv2.bin";
|
||||
const char *s2_t2_t2_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-tree2-conv1.bin";
|
||||
const char *s2_t2_t2_conv2_bin = "dla34_ctrack/layers/base-level3-tree2-tree2-conv2.bin";
|
||||
const char *s2_t2_root_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-root-conv.bin";
|
||||
const char *s3_t1_t1_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-tree1-conv1.bin";
|
||||
const char *s3_t1_t1_conv2_bin = "dla34_ctrack/layers/base-level4-tree1-tree1-conv2.bin";
|
||||
const char *s3_t1_t1_project = "dla34_ctrack/layers/base-level4-tree1-project-0.bin";
|
||||
const char *s3_t1_t2_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-tree2-conv1.bin";
|
||||
const char *s3_t1_t2_conv2_bin = "dla34_ctrack/layers/base-level4-tree1-tree2-conv2.bin";
|
||||
const char *s3_t1_root_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-root-conv.bin";
|
||||
const char *s3_t2_t1_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-tree1-conv1.bin";
|
||||
const char *s3_t2_t1_conv2_bin = "dla34_ctrack/layers/base-level4-tree2-tree1-conv2.bin";
|
||||
const char *s3_t2_t2_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-tree2-conv1.bin";
|
||||
const char *s3_t2_t2_conv2_bin = "dla34_ctrack/layers/base-level4-tree2-tree2-conv2.bin";
|
||||
const char *s3_t2_root_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-root-conv.bin";
|
||||
const char *s4_t1_conv1_bin = "dla34_ctrack/layers/base-level5-tree1-conv1.bin";
|
||||
const char *s4_t1_conv2_bin = "dla34_ctrack/layers/base-level5-tree1-conv2.bin";
|
||||
const char *s4_t1_project = "dla34_ctrack/layers/base-level5-project-0.bin";
|
||||
const char *s4_t2_conv1_bin = "dla34_ctrack/layers/base-level5-tree2-conv1.bin";
|
||||
const char *s4_t2_conv2_bin = "dla34_ctrack/layers/base-level5-tree2-conv2.bin";
|
||||
const char *s4_root_conv1_bin = "dla34_ctrack/layers/base-level5-root-conv.bin";
|
||||
|
||||
//final
|
||||
// const char *fc_bin = "dla34_ctrack/layers/output.bin";
|
||||
|
||||
const char *ida_0_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_0-proj_1-conv.bin";
|
||||
const char *ida_0_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_0_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_0-up_1.bin";
|
||||
const char *ida_0_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_0-node_1-conv.bin";
|
||||
const char *ida_0_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin";
|
||||
|
||||
const char *ida_1_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_1-conv.bin";
|
||||
const char *ida_1_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_1_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_1-up_1.bin";
|
||||
const char *ida_1_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-node_1-conv.bin";
|
||||
const char *ida_1_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_1_p_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_2-conv.bin";
|
||||
const char *ida_1_p_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin";
|
||||
const char *ida_1_up_2_deconv_bin = "dla34_ctrack/layers/dla_up-ida_1-up_2.bin";
|
||||
const char *ida_1_n_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-node_2-conv.bin";
|
||||
const char *ida_1_n_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin";
|
||||
|
||||
const char *ida_2_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_1-conv.bin";
|
||||
const char *ida_2_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_1.bin";
|
||||
const char *ida_2_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_1-conv.bin";
|
||||
const char *ida_2_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_p_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_2-conv.bin";
|
||||
const char *ida_2_p_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_up_2_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_2.bin";
|
||||
const char *ida_2_n_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_2-conv.bin";
|
||||
const char *ida_2_n_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_p_3_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_3-conv.bin";
|
||||
const char *ida_2_p_3_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin";
|
||||
const char *ida_2_up_3_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_3.bin";
|
||||
const char *ida_2_n_3_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_3-conv.bin";
|
||||
const char *ida_2_n_3_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin";
|
||||
|
||||
const char *ida_up_p_1_dcn_bin = "dla34_ctrack/layers/ida_up-proj_1-conv.bin";
|
||||
const char *ida_up_p_1_conv_bin = "dla34_ctrack/layers/ida_up-proj_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_up_up_1_deconv_bin = "dla34_ctrack/layers/ida_up-up_1.bin";
|
||||
const char *ida_up_n_1_dcn_bin = "dla34_ctrack/layers/ida_up-node_1-conv.bin";
|
||||
const char *ida_up_n_1_conv_bin = "dla34_ctrack/layers/ida_up-node_1-conv-conv_offset_mask.bin";
|
||||
const char *ida_up_p_2_dcn_bin = "dla34_ctrack/layers/ida_up-proj_2-conv.bin";
|
||||
const char *ida_up_p_2_conv_bin = "dla34_ctrack/layers/ida_up-proj_2-conv-conv_offset_mask.bin";
|
||||
const char *ida_up_up_2_deconv_bin = "dla34_ctrack/layers/ida_up-up_2.bin";
|
||||
const char *ida_up_n_2_dcn_bin = "dla34_ctrack/layers/ida_up-node_2-conv.bin";
|
||||
const char *ida_up_n_2_conv_bin = "dla34_ctrack/layers/ida_up-node_2-conv-conv_offset_mask.bin";
|
||||
|
||||
const char *hm_conv1_bin = "dla34_ctrack/layers/hm-0.bin";
|
||||
const char *hm_conv2_bin = "dla34_ctrack/layers/hm-2.bin";
|
||||
const char *wh_conv1_bin = "dla34_ctrack/layers/wh-0.bin";
|
||||
const char *wh_conv2_bin = "dla34_ctrack/layers/wh-2.bin";
|
||||
const char *reg_conv1_bin = "dla34_ctrack/layers/reg-0.bin";
|
||||
const char *reg_conv2_bin = "dla34_ctrack/layers/reg-2.bin";
|
||||
const char *track_conv1_bin = "dla34_ctrack/layers/tracking-0.bin";
|
||||
const char *track_conv2_bin = "dla34_ctrack/layers/tracking-2.bin";
|
||||
const char *dep_conv1_bin = "dla34_ctrack/layers/dep-0.bin";
|
||||
const char *dep_conv2_bin = "dla34_ctrack/layers/dep-2.bin";
|
||||
const char *rot_conv1_bin = "dla34_ctrack/layers/rot-0.bin";
|
||||
const char *rot_conv2_bin = "dla34_ctrack/layers/rot-2.bin";
|
||||
const char *dim_conv1_bin = "dla34_ctrack/layers/dim-0.bin";
|
||||
const char *dim_conv2_bin = "dla34_ctrack/layers/dim-2.bin";
|
||||
const char *a_off_conv1_bin = "dla34_ctrack/layers/amodel_offset-0.bin";
|
||||
const char *a_off_conv2_bin = "dla34_ctrack/layers/amodel_offset-2.bin";
|
||||
|
||||
const char *output_bin[]={
|
||||
"dla34_ctrack/debug/hm.bin",
|
||||
"dla34_ctrack/debug/wh.bin",
|
||||
"dla34_ctrack/debug/reg.bin",
|
||||
"dla34_ctrack/debug/tracking.bin",
|
||||
"dla34_ctrack/debug/dep.bin",
|
||||
"dla34_ctrack/debug/rot.bin",
|
||||
"dla34_ctrack/debug/dim.bin",
|
||||
"dla34_ctrack/debug/amodel_offset.bin"};
|
||||
// const char *output_bin = "dla34_ctrack/debug/base-level0-2.bin";
|
||||
int main()
|
||||
{
|
||||
|
||||
downloadWeightsifDoNotExist("dla34_ctrack/debug/input.bin", "dla34_ctrack", "https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download");
|
||||
|
||||
// Network layout
|
||||
// tk::dnn::dataDim_t dim_in0(1, 3, 512, 512, 1);
|
||||
// tk::dnn::dataDim_t dim_in1(1, 1, 512, 512, 1);
|
||||
tk::dnn::dataDim_t dim_in0(1, 16, 512, 512, 1);
|
||||
// dnnType *i0_d, *i1_d, *i2_d;
|
||||
// dnnType *i0_h, *i1_h, *i2_h;
|
||||
// checkCuda( cudaMalloc(&i0_d, dim_in0.tot()*sizeof(dnnType)) );
|
||||
// checkCuda( cudaMalloc(&i1_d, dim_in1.tot()*sizeof(dnnType)) );
|
||||
// checkCuda( cudaMalloc(&i2_d, dim_in0.tot()*sizeof(dnnType)) );
|
||||
|
||||
tk::dnn::Network net(dim_in0);
|
||||
tk::dnn::Layer *last1, *last2, *last3, *last4;
|
||||
tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2;
|
||||
// tk::dnn::Layer *pre_img, *pre_hm;
|
||||
|
||||
// //pre-img
|
||||
// readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &i0_d);
|
||||
// tk::dnn::Input *in_pre_img = new tk::dnn::Input(&net, dim_in0, i0_d);
|
||||
// tk::dnn::Conv2d pre_img_conv1(&net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true);
|
||||
// tk::dnn::Activation pre_img_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
// pre_img = &pre_img_relu;
|
||||
|
||||
// //pre-hm
|
||||
// readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &i1_d);
|
||||
// tk::dnn::Input *in_pre_hm = new tk::dnn::Input(&net, dim_in1, i1_d);
|
||||
// tk::dnn::Conv2d pre_hm_conv1(&net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true);
|
||||
// tk::dnn::Activation pre_hm_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
// pre_hm = &pre_hm_relu;
|
||||
|
||||
// // image input
|
||||
// readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &i2_d);
|
||||
// tk::dnn::Input *input_image = new tk::dnn::Input(&net, dim_in0, i2_d);
|
||||
// tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true);
|
||||
// tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
// tk::dnn::Shortcut s0_input(&net, pre_img);
|
||||
// tk::dnn::Shortcut s1_input(&net, pre_hm);
|
||||
|
||||
tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true);
|
||||
tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU);
|
||||
base1 = &relu2;
|
||||
|
||||
tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true);
|
||||
tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU);
|
||||
base2 = &relu3;
|
||||
|
||||
// level 2
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true);
|
||||
last2 = &s1_t1_conv2;
|
||||
|
||||
// get the basicblock input and apply maxpool conv2d and relu
|
||||
tk::dnn::Layer *route_s1_t1_layers[1] = { base2 };
|
||||
tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1);
|
||||
// downsample
|
||||
tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
|
||||
// project
|
||||
tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true);
|
||||
|
||||
tk::dnn::Shortcut s1_t1_s1(&net, last2);
|
||||
tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s1_t1_relu;
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s1_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s1_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 128, 56, 56
|
||||
tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 };
|
||||
tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2);
|
||||
tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true);
|
||||
tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
base3 = &s1_root_relu;
|
||||
|
||||
// level 3
|
||||
// tree 1
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true);
|
||||
last2 = &s2_t1_t1_conv2;
|
||||
|
||||
// get the basicblock input and apply maxpool conv2d and relu
|
||||
tk::dnn::Layer *route_s2_t1_t1_layers[1] = { base3 };
|
||||
tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1);
|
||||
// downsample
|
||||
tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
|
||||
last4 = &s2_t1_t1_maxpool1;
|
||||
// project
|
||||
tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true);
|
||||
|
||||
tk::dnn::Shortcut s2_t1_t1_s1(&net, last2);
|
||||
tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s2_t1_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s2_t1_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s2_t1_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 128, 56, 56
|
||||
tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 };
|
||||
tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2);
|
||||
tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s2_t1_root_relu;
|
||||
last3 = &s2_t1_root_relu;
|
||||
// tree 2
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true);
|
||||
tk::dnn::Shortcut s2_t2_t1_s1(&net, last1);
|
||||
tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s2_t2_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s2_t2_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s2_t2_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 128, 56, 56
|
||||
tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3};
|
||||
tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4);
|
||||
tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true);
|
||||
tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
base4 = &s2_t2_root_relu;
|
||||
|
||||
// level 4
|
||||
// tree 1
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true);
|
||||
last2 = &s3_t1_t1_conv2;
|
||||
|
||||
// get the basicblock input and apply maxpool conv2d and relu
|
||||
tk::dnn::Layer *route_s3_t1_t1_layers[1] = { base4 };
|
||||
tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1);
|
||||
// downsample
|
||||
tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
|
||||
last4 = &s3_t1_t1_maxpool1;
|
||||
// project
|
||||
tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true);
|
||||
|
||||
tk::dnn::Shortcut s3_t1_t1_s1(&net, last2);
|
||||
tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s3_t1_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s3_t1_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s3_t1_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 256, 56, 56
|
||||
tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 };
|
||||
tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2);
|
||||
tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s3_t1_root_relu;
|
||||
last3 = &s3_t1_root_relu;
|
||||
// tree 2
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true);
|
||||
tk::dnn::Shortcut s3_t2_t1_s1(&net, last1);
|
||||
tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s3_t2_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s3_t2_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s3_t2_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 256, 56, 56
|
||||
tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3};
|
||||
tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4);
|
||||
tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true);
|
||||
tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
base5 = &s3_t2_root_relu;
|
||||
|
||||
// level 5
|
||||
// tree 1
|
||||
tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true);
|
||||
tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true);
|
||||
last2 = &s4_t1_conv2;
|
||||
|
||||
// get the basicblock input and apply maxpool conv2d and relu
|
||||
tk::dnn::Layer *route_s4_t1_layers[1] = { base5 };
|
||||
tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1);
|
||||
// downsample
|
||||
tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
|
||||
last4 = &s4_t1_maxpool1;
|
||||
// project
|
||||
tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true);
|
||||
|
||||
tk::dnn::Shortcut s4_t1_s1(&net, last2);
|
||||
tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
last1 = &s4_t1_relu;
|
||||
|
||||
// tree 2
|
||||
tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true);
|
||||
tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true);
|
||||
|
||||
tk::dnn::Shortcut s4_t2_s1(&net, last1);
|
||||
tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
last2 = &s4_t2_relu;
|
||||
|
||||
// root
|
||||
// join last1 and net in single input 128, 56, 56
|
||||
tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 };
|
||||
tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3);
|
||||
tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true);
|
||||
tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
base6 = &s4_root_relu;
|
||||
|
||||
//final
|
||||
// tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE);
|
||||
// tk::dnn::Dense fc(&net, 1000, fc_bin);
|
||||
|
||||
//ida 0
|
||||
tk::dnn::Layer *route_ida0[1] = { base6 };
|
||||
tk::dnn::Route route_ida0_0(&net, route_ida0, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256);
|
||||
tk::dnn::Shortcut ida_0_shortcut(&net, base5);
|
||||
tk::dnn::DeformConv2d ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida1 = &ida_0_n_1_relu;
|
||||
|
||||
//ida1-1
|
||||
tk::dnn::Layer *route_ida1_layers_1[1] = { base5 };
|
||||
tk::dnn::Route route_ida1_1(&net, route_ida1_layers_1, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128);
|
||||
tk::dnn::Shortcut ida_1_shortcut1(&net, base4);
|
||||
tk::dnn::DeformConv2d ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida2_1 = &ida_1_n_1_relu;
|
||||
|
||||
//ida1-2
|
||||
tk::dnn::Layer *route_ida1_layers_2[1] = { ida1 };
|
||||
tk::dnn::Route route_ida1_2(&net, route_ida1_layers_2, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true);
|
||||
tk::dnn::Activation ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128);
|
||||
tk::dnn::Shortcut ida_1_shortcut2(&net, ida2_1);
|
||||
tk::dnn::DeformConv2d ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true);
|
||||
tk::dnn::Activation ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida2_2 = &ida_1_n_2_relu;
|
||||
|
||||
//ida2-1
|
||||
tk::dnn::Layer *route_ida2_layers_1[1] = { base4 };
|
||||
tk::dnn::Route route_ida2_1(&net, route_ida2_layers_1, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut ida_2_shortcut1(&net, base3);
|
||||
tk::dnn::DeformConv2d ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida3_1 = &ida_2_n_1_relu;
|
||||
|
||||
//ida2-2
|
||||
tk::dnn::Layer *route_ida2_layers_2[1] = { ida2_1 };
|
||||
tk::dnn::Route route_ida2_2(&net, route_ida2_layers_2, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut ida_2_shortcut2(&net, ida3_1);
|
||||
tk::dnn::DeformConv2d ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida3_2 = &ida_2_n_2_relu;
|
||||
|
||||
//ida2-3
|
||||
tk::dnn::Layer *route_ida2_layers_3[1] = { ida2_2 };
|
||||
tk::dnn::Route route_ida2_3(&net, route_ida2_layers_3, 1);
|
||||
|
||||
tk::dnn::DeformConv2d ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut ida_2_shortcut3(&net, ida3_2);
|
||||
tk::dnn::DeformConv2d ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true);
|
||||
tk::dnn::Activation ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
ida3_3 = &ida_2_n_3_relu;
|
||||
|
||||
//idaup-1
|
||||
tk::dnn::Layer *route_idaup_layers_1[1] = { ida2_2 };
|
||||
tk::dnn::Route route_idaup_1(&net, route_idaup_layers_1, 1);
|
||||
|
||||
tk::dnn::DeformConv2d idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true);
|
||||
tk::dnn::Activation idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut idaup_shortcut1(&net, ida3_3);
|
||||
tk::dnn::DeformConv2d idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true);
|
||||
tk::dnn::Activation idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
idaup_1 = &idaup_n_1_relu;
|
||||
|
||||
//idaup-2
|
||||
tk::dnn::Layer *route_idaup_layers_2[1] = { ida1 };
|
||||
tk::dnn::Route route_idaup_2(&net, route_idaup_layers_2, 1);
|
||||
|
||||
tk::dnn::DeformConv2d idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true);
|
||||
tk::dnn::Activation idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::DeConv2d idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64);
|
||||
tk::dnn::Shortcut idaup_shortcut2(&net, idaup_1);
|
||||
tk::dnn::DeformConv2d idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true);
|
||||
tk::dnn::Activation idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
|
||||
idaup_2 = &idaup_n_2_relu;
|
||||
|
||||
tk::dnn::Layer *route_1_0_layers[1] = { idaup_2 };
|
||||
|
||||
// hm
|
||||
tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false);
|
||||
tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 10, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false);
|
||||
hm->setFinal();
|
||||
|
||||
int kernel = 3;
|
||||
int pad = (kernel - 1)/2;
|
||||
tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID);
|
||||
tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX);
|
||||
hmax->setFinal();
|
||||
|
||||
// wh
|
||||
tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false);
|
||||
tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false);
|
||||
wh->setFinal();
|
||||
|
||||
// reg
|
||||
tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false);
|
||||
tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false);
|
||||
reg->setFinal();
|
||||
|
||||
// tracking
|
||||
tk::dnn::Route *route_3_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *track_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, track_conv1_bin, false);
|
||||
tk::dnn::Activation *track_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *track = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, track_conv2_bin, false);
|
||||
track->setFinal();
|
||||
|
||||
// dep
|
||||
tk::dnn::Route *route_4_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *dep_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dep_conv1_bin, false);
|
||||
tk::dnn::Activation *dep_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *dep = new tk::dnn::Conv2d(&net, 1, 1, 1, 1, 1, 0, 0, dep_conv2_bin, false);
|
||||
dep->setFinal();
|
||||
|
||||
// rot
|
||||
tk::dnn::Route *route_5_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *rot_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, rot_conv1_bin, false);
|
||||
tk::dnn::Activation *rot_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *rot = new tk::dnn::Conv2d(&net, 8, 1, 1, 1, 1, 0, 0, rot_conv2_bin, false);
|
||||
rot->setFinal();
|
||||
|
||||
// dim
|
||||
tk::dnn::Route *route_6_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *dim_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dim_conv1_bin, false);
|
||||
tk::dnn::Activation *dim_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *dim_ = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, dim_conv2_bin, false);
|
||||
dim_->setFinal();
|
||||
|
||||
// amodel_offset
|
||||
tk::dnn::Route *route_7_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
|
||||
tk::dnn::Conv2d *a_off_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, a_off_conv1_bin, false);
|
||||
tk::dnn::Activation *a_off_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
|
||||
tk::dnn::Conv2d *a_off = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, a_off_conv2_bin, false);
|
||||
a_off->setFinal();
|
||||
|
||||
|
||||
// Load input
|
||||
dnnType *data;
|
||||
dnnType *input_h;
|
||||
readBinaryFile(input_bin, dim_in0.tot(), &input_h, &data);
|
||||
//printDeviceVector(64, data, true);
|
||||
|
||||
//print network model
|
||||
net.print();
|
||||
|
||||
//convert network to tensorRT
|
||||
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_ctrack"));
|
||||
|
||||
tk::dnn::dataDim_t dim1 = dim_in0; //input dim
|
||||
printCenteredTitle(" CUDNN inference ", '=', 30);
|
||||
{
|
||||
dim1.print();
|
||||
TKDNN_TSTART
|
||||
// tk::dnn::dataDim_t dim_aus;
|
||||
// net.infer(dim_aus, nullptr);
|
||||
net.infer(dim1, data);
|
||||
TKDNN_TSTOP
|
||||
dim1.print();
|
||||
}
|
||||
|
||||
tk::dnn::dataDim_t dim2 = dim_in0;
|
||||
printCenteredTitle(" TENSORRT inference ", '=', 30);
|
||||
{
|
||||
dim2.print();
|
||||
TKDNN_TSTART
|
||||
netRT.infer(dim2, data);
|
||||
TKDNN_TSTOP
|
||||
dim2.print();
|
||||
}
|
||||
// dnnType *out, *out_h;
|
||||
// int odim = net.layers[net.num_layers-1]->output_dim.tot();
|
||||
// readBinaryFile(output_bin, odim, &out_h, &out);
|
||||
// dnnType *cudnn_out;
|
||||
// cudnn_out = net.layers[net.num_layers-1]->dstData;
|
||||
// std::cout<<"CUDNN vs correct";
|
||||
// checkResult(odim, cudnn_out, out);
|
||||
|
||||
|
||||
tk::dnn::Layer *outs[8] = { hm, wh, reg, track, dep, rot, dim_, a_off};
|
||||
int out_count = 1;
|
||||
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
|
||||
for(int i=0; i<8; i++) {
|
||||
printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30);
|
||||
|
||||
outs[i]->output_dim.print();
|
||||
|
||||
dnnType *out, *out_h;
|
||||
int odim = outs[i]->output_dim.tot();
|
||||
readBinaryFile(output_bin[i], odim, &out_h, &out);
|
||||
|
||||
dnnType *cudnn_out, *rt_out;
|
||||
cudnn_out = outs[i]->dstData;
|
||||
rt_out = (dnnType *)netRT.buffersRT[i+out_count];
|
||||
// there is the maxpool. It isn't an output but it is necessary for the process section
|
||||
if(i==0)
|
||||
out_count ++;
|
||||
|
||||
std::cout<<"CUDNN vs correct";
|
||||
ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN;
|
||||
std::cout<<"TRT vs correct";
|
||||
ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT;
|
||||
std::cout<<"CUDNN vs TRT ";
|
||||
ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
|
||||
}
|
||||
return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
#include<iostream>
|
||||
#include<vector>
|
||||
#include "tkdnn.h"
|
||||
#include "test.h"
|
||||
#include "DarknetParser.h"
|
||||
|
||||
int main() {
|
||||
std::string bin_path = "yolo4_berkeley_f1";
|
||||
std::vector<std::string> input_bins = {
|
||||
bin_path + "/layers/input.bin"
|
||||
};
|
||||
std::vector<std::string> output_bins = {
|
||||
bin_path + "/debug/layer139_out.bin",
|
||||
bin_path + "/debug/layer150_out.bin",
|
||||
bin_path + "/debug/layer161_out.bin"
|
||||
};
|
||||
std::string wgs_path = bin_path + "/layers";
|
||||
std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4_berkeley.cfg";
|
||||
std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/berkeley.names";
|
||||
downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/M7WJdGoGDaDACnN/download");
|
||||
|
||||
// parse darknet network
|
||||
tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
|
||||
net->print();
|
||||
|
||||
//convert network to tensorRT
|
||||
tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
|
||||
|
||||
int ret = testInference(input_bins, output_bins, net, netRT);
|
||||
net->releaseLayers();
|
||||
delete net;
|
||||
delete netRT;
|
||||
return ret;
|
||||
}
|
||||
Reference in New Issue
Block a user