From 64098ad2441540a3bb5d12cd42f199c54ba6b4ba Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 22 May 2020 12:51:25 +0200 Subject: [PATCH 01/33] Add CenterNet based on DLA34 for 3D, CUDNN and TensorRT work Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 + tests/dla34_cnet3d/dla34_cnet3d.cpp | 562 ++++++++++++++++++++++++++++ 2 files changed, 565 insertions(+) create mode 100644 tests/dla34_cnet3d/dla34_cnet3d.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 028d375..1ae4289 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,9 @@ target_link_libraries(test_dla34 tkDNN) add_executable(test_dla34_cnet tests/dla34_cnet/dla34_cnet.cpp) target_link_libraries(test_dla34_cnet tkDNN) +add_executable(test_dla34_cnet3d tests/dla34_cnet3d/dla34_cnet3d.cpp) +target_link_libraries(test_dla34_cnet3d tkDNN) + add_executable(test_imuodom tests/imuodom/imuodom.cpp) target_link_libraries(test_imuodom tkDNN) ################################################################################ diff --git a/tests/dla34_cnet3d/dla34_cnet3d.cpp b/tests/dla34_cnet3d/dla34_cnet3d.cpp new file mode 100644 index 0000000..ecd5693 --- /dev/null +++ b/tests/dla34_cnet3d/dla34_cnet3d.cpp @@ -0,0 +1,562 @@ +#include +#include "tkdnn.h" + +const char *input_bin = "dla34_cnet3d/debug/input.bin"; +const char *conv1_bin = "dla34_cnet3d/layers/base-base_layer-0.bin"; +const char *conv2_bin = "dla34_cnet3d/layers/base-level0-0.bin"; +const char *conv3_bin = "dla34_cnet3d/layers/base-level1-0.bin"; +// s - stage, t - tree +const char *s1_t1_conv1_bin = "dla34_cnet3d/layers/base-level2-tree1-conv1.bin"; +const char *s1_t1_conv2_bin = "dla34_cnet3d/layers/base-level2-tree1-conv2.bin"; +const char *s1_t1_project = "dla34_cnet3d/layers/base-level2-project-0.bin"; +const char *s1_t2_conv1_bin = "dla34_cnet3d/layers/base-level2-tree2-conv1.bin"; +const char *s1_t2_conv2_bin = "dla34_cnet3d/layers/base-level2-tree2-conv2.bin"; +const char *s1_root_conv1_bin = "dla34_cnet3d/layers/base-level2-root-conv.bin"; +const char *s2_t1_t1_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-tree1-conv1.bin"; +const char *s2_t1_t1_conv2_bin = "dla34_cnet3d/layers/base-level3-tree1-tree1-conv2.bin"; +const char *s2_t1_t1_project = "dla34_cnet3d/layers/base-level3-tree1-project-0.bin"; +const char *s2_t1_t2_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-tree2-conv1.bin"; +const char *s2_t1_t2_conv2_bin = "dla34_cnet3d/layers/base-level3-tree1-tree2-conv2.bin"; +const char *s2_t1_root_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-root-conv.bin"; +const char *s2_t2_t1_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-tree1-conv1.bin"; +const char *s2_t2_t1_conv2_bin = "dla34_cnet3d/layers/base-level3-tree2-tree1-conv2.bin"; +const char *s2_t2_t2_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-tree2-conv1.bin"; +const char *s2_t2_t2_conv2_bin = "dla34_cnet3d/layers/base-level3-tree2-tree2-conv2.bin"; +const char *s2_t2_root_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-root-conv.bin"; +const char *s3_t1_t1_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-tree1-conv1.bin"; +const char *s3_t1_t1_conv2_bin = "dla34_cnet3d/layers/base-level4-tree1-tree1-conv2.bin"; +const char *s3_t1_t1_project = "dla34_cnet3d/layers/base-level4-tree1-project-0.bin"; +const char *s3_t1_t2_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-tree2-conv1.bin"; +const char *s3_t1_t2_conv2_bin = "dla34_cnet3d/layers/base-level4-tree1-tree2-conv2.bin"; +const char *s3_t1_root_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-root-conv.bin"; +const char *s3_t2_t1_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-tree1-conv1.bin"; +const char *s3_t2_t1_conv2_bin = "dla34_cnet3d/layers/base-level4-tree2-tree1-conv2.bin"; +const char *s3_t2_t2_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-tree2-conv1.bin"; +const char *s3_t2_t2_conv2_bin = "dla34_cnet3d/layers/base-level4-tree2-tree2-conv2.bin"; +const char *s3_t2_root_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-root-conv.bin"; +const char *s4_t1_conv1_bin = "dla34_cnet3d/layers/base-level5-tree1-conv1.bin"; +const char *s4_t1_conv2_bin = "dla34_cnet3d/layers/base-level5-tree1-conv2.bin"; +const char *s4_t1_project = "dla34_cnet3d/layers/base-level5-project-0.bin"; +const char *s4_t2_conv1_bin = "dla34_cnet3d/layers/base-level5-tree2-conv1.bin"; +const char *s4_t2_conv2_bin = "dla34_cnet3d/layers/base-level5-tree2-conv2.bin"; +const char *s4_root_conv1_bin = "dla34_cnet3d/layers/base-level5-root-conv.bin"; + +//final +// const char *fc_bin = "dla34_cnet3d/layers/output.bin"; + +const char *ida_0_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_0-proj_1-conv.bin"; +const char *ida_0_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin"; +const char *ida_0_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_0-up_1.bin"; +const char *ida_0_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_0-node_1-conv.bin"; +const char *ida_0_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin"; + +const char *ida_1_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_1-conv.bin"; +const char *ida_1_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin"; +const char *ida_1_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_1-up_1.bin"; +const char *ida_1_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_1-conv.bin"; +const char *ida_1_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin"; +const char *ida_1_p_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_2-conv.bin"; +const char *ida_1_p_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin"; +const char *ida_1_up_2_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_1-up_2.bin"; +const char *ida_1_n_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_2-conv.bin"; +const char *ida_1_n_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin"; + +const char *ida_2_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_1-conv.bin"; +const char *ida_2_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin"; +const char *ida_2_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_1.bin"; +const char *ida_2_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_1-conv.bin"; +const char *ida_2_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin"; +const char *ida_2_p_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_2-conv.bin"; +const char *ida_2_p_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin"; +const char *ida_2_up_2_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_2.bin"; +const char *ida_2_n_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_2-conv.bin"; +const char *ida_2_n_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin"; +const char *ida_2_p_3_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_3-conv.bin"; +const char *ida_2_p_3_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin"; +const char *ida_2_up_3_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_3.bin"; +const char *ida_2_n_3_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_3-conv.bin"; +const char *ida_2_n_3_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin"; + +const char *ida_up_p_1_dcn_bin = "dla34_cnet3d/layers/ida_up-proj_1-conv.bin"; +const char *ida_up_p_1_conv_bin = "dla34_cnet3d/layers/ida_up-proj_1-conv-conv_offset_mask.bin"; +const char *ida_up_up_1_deconv_bin = "dla34_cnet3d/layers/ida_up-up_1.bin"; +const char *ida_up_n_1_dcn_bin = "dla34_cnet3d/layers/ida_up-node_1-conv.bin"; +const char *ida_up_n_1_conv_bin = "dla34_cnet3d/layers/ida_up-node_1-conv-conv_offset_mask.bin"; +const char *ida_up_p_2_dcn_bin = "dla34_cnet3d/layers/ida_up-proj_2-conv.bin"; +const char *ida_up_p_2_conv_bin = "dla34_cnet3d/layers/ida_up-proj_2-conv-conv_offset_mask.bin"; +const char *ida_up_up_2_deconv_bin = "dla34_cnet3d/layers/ida_up-up_2.bin"; +const char *ida_up_n_2_dcn_bin = "dla34_cnet3d/layers/ida_up-node_2-conv.bin"; +const char *ida_up_n_2_conv_bin = "dla34_cnet3d/layers/ida_up-node_2-conv-conv_offset_mask.bin"; + +const char *hm_conv1_bin = "dla34_cnet3d/layers/hm-0.bin"; +const char *hm_conv2_bin = "dla34_cnet3d/layers/hm-2.bin"; +const char *wh_conv1_bin = "dla34_cnet3d/layers/wh-0.bin"; +const char *wh_conv2_bin = "dla34_cnet3d/layers/wh-2.bin"; +const char *reg_conv1_bin = "dla34_cnet3d/layers/reg-0.bin"; +const char *reg_conv2_bin = "dla34_cnet3d/layers/reg-2.bin"; +const char *dep_conv1_bin = "dla34_cnet3d/layers/dep-0.bin"; +const char *dep_conv2_bin = "dla34_cnet3d/layers/dep-2.bin"; +const char *rot_conv1_bin = "dla34_cnet3d/layers/rot-0.bin"; +const char *rot_conv2_bin = "dla34_cnet3d/layers/rot-2.bin"; +const char *dim_conv1_bin = "dla34_cnet3d/layers/dim-0.bin"; +const char *dim_conv2_bin = "dla34_cnet3d/layers/dim-2.bin"; + +const char *output_bin[]={ +"dla34_cnet3d/debug/hm.bin", +"dla34_cnet3d/debug/wh.bin", +"dla34_cnet3d/debug/reg.bin", +"dla34_cnet3d/debug/dep.bin", +"dla34_cnet3d/debug/rot.bin", +"dla34_cnet3d/debug/dim.bin"}; + +int main() +{ + + // downloadWeightsifDoNotExist(input_bin, "dla34_cnet3d", "https://cloud.hipert.unimore.it/s/KRZBbCQsKAtQwpZ/download"); + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); + tk::dnn::Network net(dim); + tk::dnn::Layer *last1, *last2, *last3, *last4; + tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2; + + tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true); + tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU); + base1 = &relu2; + + tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true); + tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU); + base2 = &relu3; + + // level 2 + // tree 1 + tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true); + tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true); + last2 = &s1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s1_t1_layers[1] = { base2 }; + tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1); + // downsample + tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + // project + tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true); + + tk::dnn::Shortcut s1_t1_s1(&net, last2); + tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s1_t1_relu; + // tree 2 + tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true); + tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true); + + tk::dnn::Shortcut s1_t2_s1(&net, last1); + tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2); + tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true); + tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base3 = &s1_root_relu; + + // level 3 + // tree 1 + // tree 1 + tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true); + tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true); + last2 = &s2_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s2_t1_t1_layers[1] = { base3 }; + tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s2_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true); + + tk::dnn::Shortcut s2_t1_t1_s1(&net, last2); + tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true); + tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t1_t2_s1(&net, last1); + tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2); + tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true); + tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_root_relu; + last3 = &s2_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true); + tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s2_t2_t1_s1(&net, last1); + tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true); + tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t2_t2_s1(&net, last1); + tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t2_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4); + tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true); + tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base4 = &s2_t2_root_relu; + + // level 4 + // tree 1 + // tree 1 + tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true); + tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true); + last2 = &s3_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s3_t1_t1_layers[1] = { base4 }; + tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s3_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true); + + tk::dnn::Shortcut s3_t1_t1_s1(&net, last2); + tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true); + tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t1_t2_s1(&net, last1); + tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t1_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2); + tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true); + tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_root_relu; + last3 = &s3_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true); + tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s3_t2_t1_s1(&net, last1); + tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true); + tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t2_t2_s1(&net, last1); + tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t2_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4); + tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true); + tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base5 = &s3_t2_root_relu; + + // level 5 + // tree 1 + tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true); + tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true); + last2 = &s4_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s4_t1_layers[1] = { base5 }; + tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1); + // downsample + tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s4_t1_maxpool1; + // project + tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true); + + tk::dnn::Shortcut s4_t1_s1(&net, last2); + tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s4_t1_relu; + + // tree 2 + tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true); + tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true); + + tk::dnn::Shortcut s4_t2_s1(&net, last1); + tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s4_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 }; + tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3); + tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true); + tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base6 = &s4_root_relu; + + //final + // tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE); + // tk::dnn::Dense fc(&net, 1000, fc_bin); + + //ida 0 + tk::dnn::DeformConv2d ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true); + tk::dnn::Activation ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256); + tk::dnn::Shortcut ida_0_shortcut(&net, base5); + tk::dnn::DeformConv2d ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true); + tk::dnn::Activation ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida1 = &ida_0_n_1_relu; + + //ida1-1 + tk::dnn::Layer *route_ida1_layers_1[1] = { base5 }; + tk::dnn::Route route_ida1_1(&net, route_ida1_layers_1, 1); + + tk::dnn::DeformConv2d ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true); + tk::dnn::Activation ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut1(&net, base4); + tk::dnn::DeformConv2d ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true); + tk::dnn::Activation ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_1 = &ida_1_n_1_relu; + + //ida1-2 + tk::dnn::Layer *route_ida1_layers_2[1] = { ida1 }; + tk::dnn::Route route_ida1_2(&net, route_ida1_layers_2, 1); + + tk::dnn::DeformConv2d ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true); + tk::dnn::Activation ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut2(&net, ida2_1); + tk::dnn::DeformConv2d ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true); + tk::dnn::Activation ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_2 = &ida_1_n_2_relu; + + //ida2-1 + tk::dnn::Layer *route_ida2_layers_1[1] = { base4 }; + tk::dnn::Route route_ida2_1(&net, route_ida2_layers_1, 1); + + tk::dnn::DeformConv2d ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true); + tk::dnn::Activation ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut1(&net, base3); + tk::dnn::DeformConv2d ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true); + tk::dnn::Activation ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_1 = &ida_2_n_1_relu; + + //ida2-2 + tk::dnn::Layer *route_ida2_layers_2[1] = { ida2_1 }; + tk::dnn::Route route_ida2_2(&net, route_ida2_layers_2, 1); + + tk::dnn::DeformConv2d ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true); + tk::dnn::Activation ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut2(&net, ida3_1); + tk::dnn::DeformConv2d ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true); + tk::dnn::Activation ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_2 = &ida_2_n_2_relu; + + //ida2-3 + tk::dnn::Layer *route_ida2_layers_3[1] = { ida2_2 }; + tk::dnn::Route route_ida2_3(&net, route_ida2_layers_3, 1); + + tk::dnn::DeformConv2d ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true); + tk::dnn::Activation ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut3(&net, ida3_2); + tk::dnn::DeformConv2d ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true); + tk::dnn::Activation ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_3 = &ida_2_n_3_relu; + + //idaup-1 + tk::dnn::Layer *route_idaup_layers_1[1] = { ida2_2 }; + tk::dnn::Route route_idaup_1(&net, route_idaup_layers_1, 1); + + tk::dnn::DeformConv2d idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true); + tk::dnn::Activation idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut1(&net, ida3_3); + tk::dnn::DeformConv2d idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true); + tk::dnn::Activation idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_1 = &idaup_n_1_relu; + + //idaup-2 + tk::dnn::Layer *route_idaup_layers_2[1] = { ida1 }; + tk::dnn::Route route_idaup_2(&net, route_idaup_layers_2, 1); + + tk::dnn::DeformConv2d idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true); + tk::dnn::Activation idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut2(&net, idaup_1); + tk::dnn::DeformConv2d idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true); + tk::dnn::Activation idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_2 = &idaup_n_2_relu; + + tk::dnn::Layer *route_1_0_layers[1] = { idaup_2 }; + + // hm + tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false); + tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false); + hm->setFinal(); + int kernel = 3; + int pad = (kernel - 1)/2; + tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX); + hmax->setFinal(); + + // wh + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false); + tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false); + wh->setFinal(); + + // reg + tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false); + tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false); + reg->setFinal(); + + // dep + tk::dnn::Route *route_3_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dep_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dep_conv1_bin, false); + tk::dnn::Activation *dep_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dep = new tk::dnn::Conv2d(&net, 1, 1, 1, 1, 1, 0, 0, dep_conv2_bin, false); + dep->setFinal(); + + // rot + tk::dnn::Route *route_4_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *rot_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, rot_conv1_bin, false); + tk::dnn::Activation *rot_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *rot = new tk::dnn::Conv2d(&net, 8, 1, 1, 1, 1, 0, 0, rot_conv2_bin, false); + rot->setFinal(); + + // dim + tk::dnn::Route *route_5_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dim_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dim_conv1_bin, false); + tk::dnn::Activation *dim_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dim_ = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, dim_conv2_bin, false); + dim_->setFinal(); + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + //printDeviceVector(64, data, true); + + //print network model + net.print(); + + //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet3d")); + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + tk::dnn::Layer *outs[6] = { hm, wh, reg, dep, rot, dim_ }; + int out_count = 1; + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for(int i=0; i<6; i++) { + printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i+out_count]; + // there is the maxpool. It isn't an output but it is necessary for the process section + if(i==0) + out_count ++; + + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} From ba8c28238433922d6699abfd82ee19875bb180e5 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 27 May 2020 18:03:09 +0200 Subject: [PATCH 02/33] Add 3D CenterNet detection class Signed-off-by: Davide Sapienza --- include/tkDNN/CenternetDetection3D.h | 100 ++++++ include/tkDNN/DetectionNN3D.h | 136 +++++++ include/tkDNN/Layer.h | 10 + include/tkDNN/kernelsThrust.h | 2 + src/CenternetDetection3D.cpp | 517 +++++++++++++++++++++++++++ src/kernels/postprocessing.cu | 14 + 6 files changed, 779 insertions(+) create mode 100644 include/tkDNN/CenternetDetection3D.h create mode 100644 include/tkDNN/DetectionNN3D.h create mode 100644 src/CenternetDetection3D.cpp diff --git a/include/tkDNN/CenternetDetection3D.h b/include/tkDNN/CenternetDetection3D.h new file mode 100644 index 0000000..7b9ea7a --- /dev/null +++ b/include/tkDNN/CenternetDetection3D.h @@ -0,0 +1,100 @@ +#ifndef CENTERNETDETECTION3D_H +#define CENTERNETDETECTION3D_H + +#include "kernels.h" +#include +#include "opencv2/opencv.hpp" +#include +#include +#include // std::iota +#include // std::sort + +#include "DetectionNN3D.h" + +#include "kernelsThrust.h" + + +namespace tk { namespace dnn { + +class CenternetDetection3D : public DetectionNN3D +{ +private: + tk::dnn::dataDim_t dim; + tk::dnn::dataDim_t dim2; + tk::dnn::dataDim_t dim_hm; + tk::dnn::dataDim_t dim_wh; + tk::dnn::dataDim_t dim_reg; + tk::dnn::dataDim_t dim_dep; + tk::dnn::dataDim_t dim_rot; + tk::dnn::dataDim_t dim_dim; + float *topk_scores; + int *topk_inds_; + float *topk_ys_; + float *topk_xs_; + int *ids_d, *ids_; + + float *ones; + + float *scores, *scores_d; + int *clses, *clses_d; + int *topk_inds_d; + float *topk_ys_d; + float *topk_xs_d; + int *inttopk_xs_d, *inttopk_ys_d; + + float *xs, *ys; + + float *dep, *rot, *dim_, *wh; + float *dep_d, *rot_d, *dim_d, *wh_d; + + float *target_coords; + + #ifdef OPENCV_CUDACONTRIB + float *mean_d; + float *stddev_d; + #else + cv::Vec mean; + cv::Vec stddev; + dnnType *input; + #endif + cv::Mat r; + cv::Mat calibs; + float *d_ptrs; + + cv::Mat src; + cv::Mat dst; + cv::Mat dst2; + cv::Mat trans, trans2; + //processing + int K = 100; + int width = 128;//56; // TODO + + // pointer used in the kernels + float *src_out; + int *ids_out; + + struct threshold op; + float peakThreshold = 0.2; + float centerThreshold = 0.3; //default 0.5 + cv::Mat corners, pts3DHomo; + + std::vector detected3D; + std::vectorcls3D; + std::vector> face_id; + +public: + CenternetDetection3D() {}; + ~CenternetDetection3D() {}; + + bool init(const std::string& tensor_path, const int n_classes=3); + void preprocess(cv::Mat &frame); + void postprocess(); + cv::Mat draw(cv::Mat &frame); +}; + + +} // namespace dnn +} // namespace tk + + +#endif /*CENTERNETDETECTION_H*/ \ No newline at end of file diff --git a/include/tkDNN/DetectionNN3D.h b/include/tkDNN/DetectionNN3D.h new file mode 100644 index 0000000..111d7e2 --- /dev/null +++ b/include/tkDNN/DetectionNN3D.h @@ -0,0 +1,136 @@ +#ifndef DETECTIONNN3D_H +#define DETECTIONNN3D_H + +#include +#include +#include +#include +#include +#include "utils.h" + +#include +#include +#include + +#include "tkdnn.h" + +// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. + +#ifdef OPENCV_CUDACONTRIB +#include +#include +#endif + + +namespace tk { namespace dnn { + +class DetectionNN3D { + + protected: + tk::dnn::NetworkRT *netRT = nullptr; + dnnType *input_d; + + cv::Size originalSize; + + cv::Scalar colors[256]; + +#ifdef OPENCV_CUDACONTRIB + cv::cuda::GpuMat bgr[3]; + cv::cuda::GpuMat imagePreproc; +#else + cv::Mat bgr[3]; + cv::Mat imagePreproc; + dnnType *input; +#endif + + /** + * This method preprocess the image, before feeding it to the NN. + * + * @param frame original frame to adapt for inference. + */ + virtual void preprocess(cv::Mat &frame) = 0; + + /** + * This method postprocess the output of the NN to obtain the correct + * boundig boxes. + * + */ + virtual void postprocess() = 0; + + public: + int classes = 0; + float confThreshold = 0.3; /*threshold on the confidence of the boxes*/ + + std::vector detected; /*bounding boxes in output*/ + std::vector stats; /*keeps track of inference times (ms)*/ + std::vector classesNames; + + DetectionNN3D() {}; + ~DetectionNN3D(){}; + + /** + * Method used to inialize the class, allocate memory and compute + * needed data. + * + * @param tensor_path path to the rt file og the NN. + * @param n_classes number of classes for the given dataset. + * @return true if everything is correct, false otherwise. + */ + virtual bool init(const std::string& tensor_path, const int n_classes=3) = 0; + + /** + * Method to draw boundixg boxes and labels on a frame. + * + * @param frame orginal frame to draw bounding box on. + * @return frame with boundig boxes. + */ + virtual cv::Mat draw(cv::Mat &frame){}; + + /** + * This method performs the whole detection of the NN. + * + * @param frame frame to run detection on. + * @param save_times if set to true, preprocess, inference and postprocess times + * are saved on a csv file, otherwise not. + * @param times pointer to the output stream where to write times + */ + void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){ + if(!frame.data) + FatalError("No image data feed to detection"); + + if(save_times && times==nullptr) + FatalError("save_times set to true, but no valid ofstream given"); + + originalSize = frame.size(); + printCenteredTitle(" TENSORRT detection ", '=', 30); + { + TIMER_START + preprocess(frame); + TIMER_STOP + if(save_times) *times<input_dim; + { + dim.print(); + TIMER_START + netRT->infer(dim, input_d); + TIMER_STOP + dim.print(); + stats.push_back(t_ns); + if(save_times) *times< corners; + float prob; + + void print() + { + std::cout<<"\tcl: "<input_dim; + + const char *kitti_class_name[] = { + "person", "car", "bicycle"}; + classesNames = std::vector(kitti_class_name, std::end( kitti_class_name)); + + for(int c=0; cinput_dim.tot())); + + dim_hm = tk::dnn::dataDim_t(1, 3, 128, 128, 1); + dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1); + dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1); + dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1); + + checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) ); + checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); + checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); + for(int i =0; iinput_dim.tot())); + mean << 0.485, 0.456, 0.406; + stddev << 0.229, 0.224, 0.225; +#endif + + calibs = cv::Mat(cv::Size(4,3), CV_32F); + calibs.at(0,0) = 707.0493; + calibs.at(0,1) = 0.0; + calibs.at(0,2) = 604.0814; + calibs.at(0,3) = 45.75831; + calibs.at(1,0) = 0.0; + calibs.at(1,1) = 707.0493; + calibs.at(1,2) = 180.5066; + calibs.at(1,3) = -0.3454157; + calibs.at(2,0) = 0.0; + calibs.at(2,1) = 0.0; + calibs.at(2,2) = 1.0; + calibs.at(2,3) = 0.004981016; + + r = cv::Mat(cv::Size(3,3), CV_32F); + r.at(0,1) = 0.0; + r.at(1,0) = 0.0; + r.at(1,1) = 1.0; + r.at(1,2) = 0.0; + r.at(2,1) = 0.0; + + corners = cv::Mat(cv::Size(8,3), CV_32F); + corners.at(1,0) = 0.0; + corners.at(1,1) = 0.0; + corners.at(1,2) = 0.0; + corners.at(1,3) = 0.0; + + pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F); + pts3DHomo.at(3,0) = 1.0; + pts3DHomo.at(3,1) = 1.0; + pts3DHomo.at(3,2) = 1.0; + pts3DHomo.at(3,3) = 1.0; + pts3DHomo.at(3,4) = 1.0; + pts3DHomo.at(3,5) = 1.0; + pts3DHomo.at(3,6) = 1.0; + pts3DHomo.at(3,7) = 1.0; + + checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) ); + + // Alloc array used in the kernel + checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); + + dst2.at(0,0)=width * 0.5; + dst2.at(0,1)=width * 0.5; + dst2.at(1,0)=width * 0.5; + dst2.at(1,1)=width * 0.5 + width * -0.5; + + dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); + dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); + + face_id.push_back({0,1,5,4}); + face_id.push_back({1,2,6, 5}); + face_id.push_back({2,3,7,6}); + face_id.push_back({3,0,4,7}); + // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); +} + +void CenternetDetection3D::preprocess(cv::Mat &frame){ + // -----------------------------------pre-process ------------------------------------------ + + // auto start_t = std::chrono::steady_clock::now(); + // auto step_t = std::chrono::steady_clock::now(); + // auto end_t = std::chrono::steady_clock::now(); + cv::Size sz = originalSize; + // std::cout<<"image: "< 0 + + src.at(0,0)=c[0]; + src.at(0,1)=c[1]; + src.at(1,0)=c[0]; + src.at(1,1)=c[1] + s[0] * -0.5; + dst.at(0,0)=netRT->input_dim.w * 0.5; + dst.at(0,1)=netRT->input_dim.h * 0.5; + dst.at(1,0)=netRT->input_dim.w * 0.5; + dst.at(1,1)=netRT->input_dim.h * 0.5 + netRT->input_dim.w * -0.5; + + src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); + src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); + dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); + dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); + + trans = cv::getAffineTransform( src, dst ); + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME gett affine trans: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + trans2 = cv::getAffineTransform( dst2, src ); + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + } + sz_old = sz; +#ifdef OPENCV_CUDACONTRIB + std::cout<<"OPENCV CPMTROB\n"; + cv::cuda::GpuMat im_Orig; + cv::cuda::GpuMat imageF1_d, imageF2_d; + + im_Orig = cv::cuda::GpuMat(frame); + // cv::cuda::resize (im_Orig, imageF1_d, cv::Size(new_width, new_height)); + imageF1_d = im_Orig; + checkCuda( cudaDeviceSynchronize() ); + + sz = imageF1_d.size(); + // std::cout<<"size: "<(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR ); + checkCuda( cudaDeviceSynchronize() ); + + imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0); + checkCuda( cudaDeviceSynchronize() ); + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME convert: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + dim2 = dim; + cv::cuda::GpuMat bgr[3]; + cv::cuda::split(imageF1_d,bgr);//split source + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME split: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + for(int i=0; i(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + checkCuda(cudaMemcpy(input_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; +#else + std::cout<<"NO OPENCV CPMTROB\n"; + cv::Mat imageF; + // resize(frame, imageF, cv::Size(new_width, new_height)); + imageF = frame; + sz = imageF.size(); + // std::cout<<"size: "<(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + cv::Mat trans = cv::getAffineTransform( src, dst ); + cv::warpAffine(imageF, imageF, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR ); + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME warpAffine: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + sz = imageF.size(); + // std::cout<<"size: "<(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + dim2 = dim; + //split channels + cv::Mat bgr[3]; + cv::split(imageF,bgr);//split source + for(int i=0; i<3; i++){ + bgr[i] = bgr[i] - mean[i]; + bgr[i] = bgr[i] / stddev[i]; + } + + //write channels + for(int i=0; ibuffersRT[1]; + rt_out[1] = (dnnType *)netRT->buffersRT[2]; + rt_out[2] = (dnnType *)netRT->buffersRT[3]; + rt_out[3] = (dnnType *)netRT->buffersRT[4]; + rt_out[4] = (dnnType *)netRT->buffersRT[5]; + rt_out[5] = (dnnType *)netRT->buffersRT[6]; + rt_out[6] = (dnnType *)netRT->buffersRT[7]; + + // ------------------------------------ process -------------------------------------------- + activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot()); + checkCuda( cudaDeviceSynchronize() ); + + // output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1. + activationSIGMOIDForward(rt_out[4], rt_out[4], dim_dep.tot()); + checkCuda( cudaDeviceSynchronize() ); + transformDep(ones, ones + dim_dep.tot(), rt_out[4], rt_out[4] + dim_dep.tot()); + checkCuda( cudaDeviceSynchronize() ); + + subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op); + + // ----------- nms end + // ----------- topk + + if(K > dim_hm.h * dim_hm.w){ + printf ("Error topk (K is too large)\n"); + return; + } + + checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) ); + + sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d); + checkCuda( cudaDeviceSynchronize() ); + + topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d); + checkCuda( cudaDeviceSynchronize() ); + + checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) ); + + topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d); + + checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); + + checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); + + // ----------- topk end + + topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out); + // checkCuda( cudaDeviceSynchronize() ); + + + getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[4], dep_d, ids_out); + checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[5], rot_d, ids_out); + checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[6], dim_d, ids_out); + checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_wh.c, dim_wh.h * dim_wh.w, rt_out[2], wh_d, ids_out); + checkCuda( cudaMemcpy(wh, wh_d, K * dim_wh.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + checkCuda( cudaMemcpy(xs, topk_xs_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(ys, topk_ys_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + + // ---------------------------------- post-process ----------------------------------------- + + // ddd_post_process_2d + cv::Mat new_pt1(cv::Size(1,2), CV_32F); + cv::Mat new_pt2(cv::Size(1,2), CV_32F); + + for(int i = 0; i(0,0)=static_cast(trans2.at(0,0))*xs[i] + + static_cast(trans2.at(0,1))*ys[i] + + static_cast(trans2.at(0,2))*1.0; + new_pt1.at(0,1)=static_cast(trans2.at(1,0))*xs[i] + + static_cast(trans2.at(1,1))*ys[i] + + static_cast(trans2.at(1,2))*1.0; + + new_pt2.at(0,0)=static_cast(trans2.at(0,0))*wh[i] + + static_cast(trans2.at(0,1))*wh[K+i] + + static_cast(trans2.at(0,2))*1.0; + new_pt2.at(0,1)=static_cast(trans2.at(1,0))*wh[i] + + static_cast(trans2.at(1,1))*wh[K+i] + + static_cast(trans2.at(1,2))*1.0; + + target_coords[i*4] = new_pt1.at(0,0); + target_coords[i*4+1] = new_pt1.at(0,1); + target_coords[i*4+2] = new_pt2.at(0,0); + target_coords[i*4+3] = new_pt2.at(0,1); + } + + float alpha; + float x, y, z, rot_y; + detected3D.clear(); + for(int i = 0; i rot[5*K + j]) + alpha = std::atan2(rot[2*K + j], rot[3*K + j]) -0.5 * M_PI; + else + alpha = std::atan2(rot[6*K + j], rot[7*K + j]) +0.5 * M_PI; + + // unproject_2d_to_3d + z = dep[j] - calibs.at(2,3);// z = depth - P[2, 3] + x = (target_coords[j*4] * dep[j] - calibs.at(0,3) - calibs.at(0,2) * z) / calibs.at(0,0); + y = (target_coords[j*4+1] * dep[j] - calibs.at(1,3) - calibs.at(1,2) * z) / calibs.at(1,1) + (dim_[j] / 2); + // alpha2rot_y + rot_y = (alpha + std::atan2(target_coords[j*4] - calibs.at(0,2), calibs.at(0,0))); + if(rot_y>M_PI) + rot_y -= 2*M_PI; + if(rot_y peakThreshold) { + if(scores[j] > centerThreshold) { + if(z>0) { + // compute_box_3d + r.at(0,0) = std::cos(rot_y); + r.at(0,2) = std::sin(rot_y); + r.at(2,0) = -std::sin(rot_y); + r.at(2,2) = std::cos(rot_y); + + corners.at(0,0) = dim_[2*K+j]/2; + corners.at(0,1) = dim_[2*K+j]/2; + corners.at(0,2) = -dim_[2*K+j]/2; + corners.at(0,3) = -dim_[2*K+j]/2; + corners.at(0,4) = dim_[2*K+j]/2; + corners.at(0,5) = dim_[2*K+j]/2; + corners.at(0,6) = -dim_[2*K+j]/2; + corners.at(0,7) = -dim_[2*K+j]/2; + + corners.at(1,4) = -dim_[j]; + corners.at(1,5) = -dim_[j]; + corners.at(1,6) = -dim_[j]; + corners.at(1,7) = -dim_[j]; + + corners.at(2,0) = dim_[K+j]/2; + corners.at(2,1) = -dim_[K+j]/2; + corners.at(2,2) = -dim_[K+j]/2; + corners.at(2,3) = dim_[K+j]/2; + corners.at(2,4) = dim_[K+j]/2; + corners.at(2,5) = -dim_[K+j]/2; + corners.at(2,6) = -dim_[K+j]/2; + corners.at(2,7) = dim_[K+j]/2; + cv::Mat aus = r * corners; + + for(int k=0; k<8; k++) { + aus.at(0,k) += x; + aus.at(1,k) += y; + aus.at(2,k) += z; + } + // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); + for(int k1=0; k1<3; k1++) { + for(int k2=0; k2<8; k2++) + pts3DHomo.at(k1,k2) = aus.at(k1,k2); + } + aus.release(); + aus = calibs * pts3DHomo; + + tk::dnn::box3D res; + for(int k=0; k<8; k++) { + res.corners.push_back(aus.at(0,k) / aus.at(2,k)); + res.corners.push_back(aus.at(1,k) / aus.at(2,k)); + } + res.cl = i; + res.prob = scores[j]; + res.print(); + detected3D.push_back(res); + } + } + } + } + } +} + +cv::Mat CenternetDetection3D::draw(cv::Mat &frame) { + tk::dnn::box3D b; + int x0, w, x1, y0, h, y1; + int objClass; + std::string det_class; + + int baseline = 0; + float font_scale = 0.5; + int thickness = 2; + + // draw dets + for(int i=0; i=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), + b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + colors[b.cl], 2); + if(ind_f == 0) { + cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), + b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), + b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); + cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), + b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), + b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + } + } + } + // draw label + cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); + cv::rectangle(frame, cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1)), + cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), + (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); + cv::putText(frame, classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); + } + return frame; +} + +}} + + diff --git a/src/kernels/postprocessing.cu b/src/kernels/postprocessing.cu index 3510200..88dbcc1 100644 --- a/src/kernels/postprocessing.cu +++ b/src/kernels/postprocessing.cu @@ -1,5 +1,11 @@ #include "kernelsThrust.h" +void transformDep(float *src_begin, float *src_end, float *dst_begin, float *dst_end) { + int e = exp(-6); + thrust::transform(thrust::device, dst_begin, dst_end, thrust::make_constant_iterator(e), dst_begin, thrust::plus()); + thrust::transform(thrust::device, src_begin, src_end, dst_begin, dst_begin, thrust::divides()); + thrust::transform(thrust::device, dst_begin, dst_end, thrust::make_constant_iterator(-1.0), dst_begin, thrust::plus()); +} void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op){ thrust::transform(thrust::device, src_begin, src_end, src2_begin, src_out, op); @@ -51,6 +57,14 @@ void topKxyAddOffset(int * ids_begin, const int K, const int size, thrust::transform(thrust::device, intys_begin, intys_begin + K, src_out, ys_begin, thrust::plus()); } +void getRecordsFromTopKId(int * ids_begin, const int K, const int ch, const int size, dnnType *src_begin, float *src_out, int *ids_out) { + for(int i=0; i()); + thrust::gather(thrust::device, ids_out, ids_out + K, src_begin, src_out+i*K); + } +} + void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out){ From 6bdf47bae60acb200d40c33b9c6a85b2013b920f Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 27 May 2020 18:05:38 +0200 Subject: [PATCH 03/33] Add 3D demo program Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 ++ demo/demo/demo3D.cpp | 103 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 demo/demo/demo3D.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ae4289..0e75e3b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,6 +159,9 @@ target_link_libraries(map_demo tkDNN) add_executable(demo demo/demo/demo.cpp) target_link_libraries(demo tkDNN) +add_executable(demo3D demo/demo/demo3D.cpp) +target_link_libraries(demo3D tkDNN) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp new file mode 100644 index 0000000..e838395 --- /dev/null +++ b/demo/demo/demo3D.cpp @@ -0,0 +1,103 @@ +#include +#include +#include /* srand, rand */ +#include +#include + +#include "CenternetDetection3D.h" + +bool gRun; +bool SAVE_RESULT = false; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + std::cout<<"detection\n"; + signal(SIGINT, sig_handler); + + + std::string net = "dla34_cnet3d_fp32.rt"; + if(argc > 1) + net = argv[1]; + std::string input = "../demo/yolo_test.mp4"; + if(argc > 2) + input = argv[2]; + char ntype = 'c'; + if(argc > 3) + ntype = argv[3][0]; + int n_classes = 3; + if(argc > 4) + n_classes = atoi(argv[4]); + + tk::dnn::CenternetDetection3D cnet; + + tk::dnn::DetectionNN3D *detNN; + + switch(ntype) + { + case 'c': + detNN = &cnet; + break; + default: + FatalError("Network type not allowed (3rd parameter)\n"); + } + + detNN->init(net, n_classes); + + gRun = true; + + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + + cv::Mat frame; + cv::Mat dnn_input; + cv::namedWindow("detection", cv::WINDOW_NORMAL); + + std::vector detected_bbox; + + while(gRun) { + cap >> frame; + if(!frame.data) { + break; + } + + // this will be resized to the net format + dnn_input = frame.clone(); + + //inference + detNN->update(dnn_input); + frame = detNN->draw(frame); + + cv::imshow("detection", frame); + cv::waitKey(1); + if(SAVE_RESULT) + resultVideo << frame; + } + + std::cout<<"detection end\n"; + double mean = 0; + + std::cout<stats.begin(), detNN->stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n"; + for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); + std::cout<<"Avg: "< Date: Wed, 27 May 2020 18:06:26 +0200 Subject: [PATCH 04/33] Update README Signed-off-by: Davide Sapienza --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index aeb3e02..d700d02 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,19 @@ N.b. By default it is used FP32 inference ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) +### Run the 3D demo + +To run the 3D object detection demo follow these steps (example with CenterNet based on DLA34): +``` +rm dla34_cnet3d_fp32.rt # be sure to delete(or move) old tensorRT files +./test_dla34_cnet3d # run the yolo test (is slow) +./demo3D dla34_cnet3d_fp32.rt ../demo/yolo_test.mp4 c +``` +The demo3D program takes the same parameters of the demo program: +``` +./demo +``` + ### FP16 inference To run the an object detection demo with FP16 inference follow these steps (example with yolov3): From 7a677d5c10ed913593fefa38e6d4f3fc4065008e Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 29 May 2020 14:54:36 +0200 Subject: [PATCH 05/33] Add CenterNet based on Resnet101 for 3D, CUDNN and TensorRT work Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 + tests/resnet101_cnet3d/resnet101_cnet3d.cpp | 443 ++++++++++++++++++++ 2 files changed, 446 insertions(+) create mode 100644 tests/resnet101_cnet3d/resnet101_cnet3d.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e75e3b..e5a2b26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,9 @@ target_link_libraries(test_bdd-csresnext50-panet-spp tkDNN) add_executable(test_resnet101_cnet tests/resnet101_cnet/resnet101_cnet.cpp) target_link_libraries(test_resnet101_cnet tkDNN) +add_executable(test_resnet101_cnet3d tests/resnet101_cnet3d/resnet101_cnet3d.cpp) +target_link_libraries(test_resnet101_cnet3d tkDNN) + add_executable(test_dla34 tests/dla34/dla34.cpp) target_link_libraries(test_dla34 tkDNN) diff --git a/tests/resnet101_cnet3d/resnet101_cnet3d.cpp b/tests/resnet101_cnet3d/resnet101_cnet3d.cpp new file mode 100644 index 0000000..0089a19 --- /dev/null +++ b/tests/resnet101_cnet3d/resnet101_cnet3d.cpp @@ -0,0 +1,443 @@ +#include + +#include "kernels.h" +#include "Yolo3Detection.h" +#include "tkdnn.h" +#include +#include // std::iota +#include // std::sort +// #include "utils.h" + +const char *input_bin = "resnet101_cnet3d/debug/input.bin"; +const char *conv1_bin = "resnet101_cnet3d/layers/conv1.bin"; + +//layer1 +const char *layer1_bin[]={ +"resnet101_cnet3d/layers/layer1-0-conv1.bin", +"resnet101_cnet3d/layers/layer1-0-conv2.bin", +"resnet101_cnet3d/layers/layer1-0-conv3.bin", +"resnet101_cnet3d/layers/layer1-0-downsample-0.bin", + +"resnet101_cnet3d/layers/layer1-1-conv1.bin", +"resnet101_cnet3d/layers/layer1-1-conv2.bin", +"resnet101_cnet3d/layers/layer1-1-conv3.bin", + +"resnet101_cnet3d/layers/layer1-2-conv1.bin", +"resnet101_cnet3d/layers/layer1-2-conv2.bin", +"resnet101_cnet3d/layers/layer1-2-conv3.bin"}; + + +//layer2 +const char *layer2_bin[]={ +"resnet101_cnet3d/layers/layer2-0-conv1.bin", +"resnet101_cnet3d/layers/layer2-0-conv2.bin", +"resnet101_cnet3d/layers/layer2-0-conv3.bin", +"resnet101_cnet3d/layers/layer2-0-downsample-0.bin", + +"resnet101_cnet3d/layers/layer2-1-conv1.bin", +"resnet101_cnet3d/layers/layer2-1-conv2.bin", +"resnet101_cnet3d/layers/layer2-1-conv3.bin", + +"resnet101_cnet3d/layers/layer2-2-conv1.bin", +"resnet101_cnet3d/layers/layer2-2-conv2.bin", +"resnet101_cnet3d/layers/layer2-2-conv3.bin", + +"resnet101_cnet3d/layers/layer2-3-conv1.bin", +"resnet101_cnet3d/layers/layer2-3-conv2.bin", +"resnet101_cnet3d/layers/layer2-3-conv3.bin" +}; +//layer3 +const char *layer3_bin[]={ +"resnet101_cnet3d/layers/layer3-0-conv1.bin", +"resnet101_cnet3d/layers/layer3-0-conv2.bin", +"resnet101_cnet3d/layers/layer3-0-conv3.bin", +"resnet101_cnet3d/layers/layer3-0-downsample-0.bin", + +"resnet101_cnet3d/layers/layer3-1-conv1.bin", +"resnet101_cnet3d/layers/layer3-1-conv2.bin", +"resnet101_cnet3d/layers/layer3-1-conv3.bin", + +"resnet101_cnet3d/layers/layer3-2-conv1.bin", +"resnet101_cnet3d/layers/layer3-2-conv2.bin", +"resnet101_cnet3d/layers/layer3-2-conv3.bin", + +"resnet101_cnet3d/layers/layer3-3-conv1.bin", +"resnet101_cnet3d/layers/layer3-3-conv2.bin", +"resnet101_cnet3d/layers/layer3-3-conv3.bin", + +"resnet101_cnet3d/layers/layer3-4-conv1.bin", +"resnet101_cnet3d/layers/layer3-4-conv2.bin", +"resnet101_cnet3d/layers/layer3-4-conv3.bin", + +"resnet101_cnet3d/layers/layer3-5-conv1.bin", +"resnet101_cnet3d/layers/layer3-5-conv2.bin", +"resnet101_cnet3d/layers/layer3-5-conv3.bin", + +"resnet101_cnet3d/layers/layer3-6-conv1.bin", +"resnet101_cnet3d/layers/layer3-6-conv2.bin", +"resnet101_cnet3d/layers/layer3-6-conv3.bin", + +"resnet101_cnet3d/layers/layer3-7-conv1.bin", +"resnet101_cnet3d/layers/layer3-7-conv2.bin", +"resnet101_cnet3d/layers/layer3-7-conv3.bin", + +"resnet101_cnet3d/layers/layer3-8-conv1.bin", +"resnet101_cnet3d/layers/layer3-8-conv2.bin", +"resnet101_cnet3d/layers/layer3-8-conv3.bin", + +"resnet101_cnet3d/layers/layer3-9-conv1.bin", +"resnet101_cnet3d/layers/layer3-9-conv2.bin", +"resnet101_cnet3d/layers/layer3-9-conv3.bin", + +"resnet101_cnet3d/layers/layer3-10-conv1.bin", +"resnet101_cnet3d/layers/layer3-10-conv2.bin", +"resnet101_cnet3d/layers/layer3-10-conv3.bin", + +"resnet101_cnet3d/layers/layer3-11-conv1.bin", +"resnet101_cnet3d/layers/layer3-11-conv2.bin", +"resnet101_cnet3d/layers/layer3-11-conv3.bin", + +"resnet101_cnet3d/layers/layer3-12-conv1.bin", +"resnet101_cnet3d/layers/layer3-12-conv2.bin", +"resnet101_cnet3d/layers/layer3-12-conv3.bin", + +"resnet101_cnet3d/layers/layer3-13-conv1.bin", +"resnet101_cnet3d/layers/layer3-13-conv2.bin", +"resnet101_cnet3d/layers/layer3-13-conv3.bin", + +"resnet101_cnet3d/layers/layer3-14-conv1.bin", +"resnet101_cnet3d/layers/layer3-14-conv2.bin", +"resnet101_cnet3d/layers/layer3-14-conv3.bin", + +"resnet101_cnet3d/layers/layer3-15-conv1.bin", +"resnet101_cnet3d/layers/layer3-15-conv2.bin", +"resnet101_cnet3d/layers/layer3-15-conv3.bin", + +"resnet101_cnet3d/layers/layer3-16-conv1.bin", +"resnet101_cnet3d/layers/layer3-16-conv2.bin", +"resnet101_cnet3d/layers/layer3-16-conv3.bin", + +"resnet101_cnet3d/layers/layer3-17-conv1.bin", +"resnet101_cnet3d/layers/layer3-17-conv2.bin", +"resnet101_cnet3d/layers/layer3-17-conv3.bin", + +"resnet101_cnet3d/layers/layer3-18-conv1.bin", +"resnet101_cnet3d/layers/layer3-18-conv2.bin", +"resnet101_cnet3d/layers/layer3-18-conv3.bin", + +"resnet101_cnet3d/layers/layer3-19-conv1.bin", +"resnet101_cnet3d/layers/layer3-19-conv2.bin", +"resnet101_cnet3d/layers/layer3-19-conv3.bin", + +"resnet101_cnet3d/layers/layer3-20-conv1.bin", +"resnet101_cnet3d/layers/layer3-20-conv2.bin", +"resnet101_cnet3d/layers/layer3-20-conv3.bin", + +"resnet101_cnet3d/layers/layer3-21-conv1.bin", +"resnet101_cnet3d/layers/layer3-21-conv2.bin", +"resnet101_cnet3d/layers/layer3-21-conv3.bin", + +"resnet101_cnet3d/layers/layer3-22-conv1.bin", +"resnet101_cnet3d/layers/layer3-22-conv2.bin", +"resnet101_cnet3d/layers/layer3-22-conv3.bin"}; + + +//layer4 +const char *layer4_bin[]={ +"resnet101_cnet3d/layers/layer4-0-conv1.bin", +"resnet101_cnet3d/layers/layer4-0-conv2.bin", +"resnet101_cnet3d/layers/layer4-0-conv3.bin", +"resnet101_cnet3d/layers/layer4-0-downsample-0.bin", + +"resnet101_cnet3d/layers/layer4-1-conv1.bin", +"resnet101_cnet3d/layers/layer4-1-conv2.bin", +"resnet101_cnet3d/layers/layer4-1-conv3.bin", + +"resnet101_cnet3d/layers/layer4-2-conv1.bin", +"resnet101_cnet3d/layers/layer4-2-conv2.bin", +"resnet101_cnet3d/layers/layer4-2-conv3.bin"}; + +const char *d_conv1_bin = "resnet101_cnet3d/layers/deconv_layers-0-conv_offset_mask.bin"; +const char *deform1_bin = "resnet101_cnet3d/layers/deconv_layers-0.bin"; +const char *deconv1_bin = "resnet101_cnet3d/layers/deconv_layers-3.bin"; + +const char *d_conv2_bin = "resnet101_cnet3d/layers/deconv_layers-6-conv_offset_mask.bin"; +const char *deform2_bin = "resnet101_cnet3d/layers/deconv_layers-6.bin"; +const char *deconv2_bin = "resnet101_cnet3d/layers/deconv_layers-9.bin"; + +const char *d_conv3_bin = "resnet101_cnet3d/layers/deconv_layers-12-conv_offset_mask.bin"; +const char *deform3_bin = "resnet101_cnet3d/layers/deconv_layers-12.bin"; +const char *deconv3_bin = "resnet101_cnet3d/layers/deconv_layers-15.bin"; + +const char *hm_conv1_bin = "resnet101_cnet3d/layers/hm-0.bin"; +const char *hm_conv2_bin = "resnet101_cnet3d/layers/hm-2.bin"; +const char *wh_conv1_bin = "resnet101_cnet3d/layers/wh-0.bin"; +const char *wh_conv2_bin = "resnet101_cnet3d/layers/wh-2.bin"; +const char *reg_conv1_bin = "resnet101_cnet3d/layers/reg-0.bin"; +const char *reg_conv2_bin = "resnet101_cnet3d/layers/reg-2.bin"; +const char *dep_conv1_bin = "resnet101_cnet3d/layers/dep-0.bin"; +const char *dep_conv2_bin = "resnet101_cnet3d/layers/dep-2.bin"; +const char *rot_conv1_bin = "resnet101_cnet3d/layers/rot-0.bin"; +const char *rot_conv2_bin = "resnet101_cnet3d/layers/rot-2.bin"; +const char *dim_conv1_bin = "resnet101_cnet3d/layers/dim-0.bin"; +const char *dim_conv2_bin = "resnet101_cnet3d/layers/dim-2.bin"; +//final +const char *fc_bin = "resnet101_cnet3d/layers/fc.bin"; + +const char *output_bin[]={ +"resnet101_cnet3d/debug/hm.bin", +"resnet101_cnet3d/debug/wh.bin", +"resnet101_cnet3d/debug/reg.bin", +"resnet101_cnet3d/debug/dep.bin", +"resnet101_cnet3d/debug/rot.bin", +"resnet101_cnet3d/debug/dim.bin"}; + +int main() +{ + // downloadWeightsifDoNotExist(input_bin, "resnet101_cnet3d", "https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download"); + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); + tk::dnn::Network net(dim); + + tk::dnn::Conv2d conv1(&net, 64, 7, 7, 2, 2, 3, 3, conv1_bin, true); + tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Pooling maxpool4(&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); + + + //layer 1 + int id_layer1_bin = 0; + tk::dnn::Layer *last = &maxpool4; + for(int i=0; i<3;i++) + { + tk::dnn::Conv2d *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 64, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true); + tk::dnn::Activation *relu1_0_1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv2 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, layer1_bin[id_layer1_bin++], true); + tk::dnn::Activation *relu1_0_2 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true); + if(i==0) { + tk::dnn::Layer *route_1_0_layers[1] = { last }; + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true); + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, layer1_0_conv3); + } else { + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, last); + } + tk::dnn::Activation *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + last = layer1_0_relu; + } + + // layer 2 + int id_layer2_bin = 0; + for(int i=0; i<4;i++) + { + tk::dnn::Conv2d *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 128, 1, 1, 1, 1, 0, 0, layer2_bin[id_layer2_bin++], true); + tk::dnn::Activation *relu1_0_1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv2; + if(i==0) + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 128, 3, 3, 2, 2, 1, 1, layer2_bin[id_layer2_bin++], true); + else + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 128, 3, 3, 1, 1, 1, 1, layer2_bin[id_layer2_bin++], true); + + tk::dnn::Activation *relu1_0_2 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 512, 1, 1, 1, 1, 0, 0, layer2_bin[id_layer2_bin++], true); + if(i==0) + { + tk::dnn::Layer *route_1_0_layers[1] = { last }; + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 512, 1, 1, 2, 2, 0, 0, layer2_bin[id_layer2_bin++], true); + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, layer1_0_conv3); + } + else + { + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, last); + } + tk::dnn::Activation *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + last = layer1_0_relu; + } + + // layer 3 + int id_layer3_bin = 0; + for(int i=0; i<23;i++) + { + tk::dnn::Conv2d *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer3_bin[id_layer3_bin++], true); + tk::dnn::Activation *relu1_0_1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv2; + if(i==0) + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 256, 3, 3, 2, 2, 1, 1, layer3_bin[id_layer3_bin++], true); + else + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, layer3_bin[id_layer3_bin++], true); + + tk::dnn::Activation *relu1_0_2 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 1024, 1, 1, 1, 1, 0, 0, layer3_bin[id_layer3_bin++], true); + if(i==0) + { + tk::dnn::Layer *route_1_0_layers[1] = { last }; + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 1024, 1, 1, 2, 2, 0, 0, layer3_bin[id_layer3_bin++], true); + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, layer1_0_conv3); + } + else + { + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, last); + } + tk::dnn::Activation *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + last = layer1_0_relu; + } + + // layer 4 + int id_layer4_bin = 0; + for(int i=0; i<3;i++) + { + tk::dnn::Conv2d *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 512, 1, 1, 1, 1, 0, 0, layer4_bin[id_layer4_bin++], true); + tk::dnn::Activation *relu1_0_1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv2; + if(i==0) + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 512, 3, 3, 2, 2, 1, 1, layer4_bin[id_layer4_bin++], true); + else + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 512, 3, 3, 1, 1, 1, 1, layer4_bin[id_layer4_bin++], true); + + tk::dnn::Activation *relu1_0_2 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 2048, 1, 1, 1, 1, 0, 0, layer4_bin[id_layer4_bin++], true); + if(i==0) + { + tk::dnn::Layer *route_1_0_layers[1] = { last }; + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 2048, 1, 1, 2, 2, 0, 0, layer4_bin[id_layer4_bin++], true); + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, layer1_0_conv3); + } + else + { + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, last); + } + tk::dnn::Activation *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + last = layer1_0_relu; + } + + tk::dnn::DeformConv2d *layer0_deform1 = new tk::dnn::DeformConv2d(&net, 256, 1, 3, 3, 1, 1, 1, 1, deform1_bin, d_conv1_bin, true); + tk::dnn::Activation *layer0_deform1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d *layer0_deconv1 = new tk::dnn::DeConv2d(&net, 256, 4, 4, 2, 2, 1, 1, deconv1_bin, true); + tk::dnn::Activation *layer0_deconv1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::DeformConv2d *layer1_deform1 = new tk::dnn::DeformConv2d(&net, 128, 1, 3, 3, 1, 1, 1, 1, deform2_bin, d_conv2_bin, true); + tk::dnn::Activation *layer1_deform1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d *layer1_deconv1 = new tk::dnn::DeConv2d(&net, 128, 4, 4, 2, 2, 1, 1, deconv2_bin, true); + tk::dnn::Activation *layer1_deconv1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::DeformConv2d *layer2_deform1 = new tk::dnn::DeformConv2d(&net, 64, 1, 3, 3, 1, 1, 1, 1, deform3_bin, d_conv3_bin, true); + tk::dnn::Activation *layer2_deform1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d *layer2_deconv1 = new tk::dnn::DeConv2d(&net, 64, 4, 4, 2, 2, 1, 1, deconv3_bin, true); + tk::dnn::Activation *layer2_deconv1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Layer *route_1_0_layers[1] = { layer2_deconv1_relu }; + tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false); + tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false); + hm->setFinal(); + int kernel = 3; + int pad = (kernel - 1)/2; + tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX); + hmax->setFinal(); + + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false); + tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false); + wh->setFinal(); + + tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false); + tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false); + reg->setFinal(); + + // dep + tk::dnn::Route *route_3_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dep_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, dep_conv1_bin, false); + tk::dnn::Activation *dep_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dep = new tk::dnn::Conv2d(&net, 1, 1, 1, 1, 1, 0, 0, dep_conv2_bin, false); + dep->setFinal(); + + // rot + tk::dnn::Route *route_4_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *rot_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, rot_conv1_bin, false); + tk::dnn::Activation *rot_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *rot = new tk::dnn::Conv2d(&net, 8, 1, 1, 1, 1, 0, 0, rot_conv2_bin, false); + rot->setFinal(); + + // dim + tk::dnn::Route *route_5_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dim_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, dim_conv1_bin, false); + tk::dnn::Activation *dim_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dim_ = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, dim_conv2_bin, false); + dim_->setFinal(); + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + // printDeviceVector(64, data, true); + + //print network model + net.print(); + + //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("resnet101_cnet3d")); + + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + + // printDeviceVector(64, cudnn_out, true); + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + tk::dnn::Layer *outs[6] = { hm, wh, reg, dep, rot, dim_ }; + int out_count = 1; + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for(int i=0; i<6; i++) { + printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + // std::cout<<"OUTPUT BIN:\n"; + // printDeviceVector(odim, cudnn_out, true); + // std::cout<<"FILE BIN:\n"; + // printDeviceVector(odim, out, true); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i+out_count]; + // there is the maxpool. It isn't an output but it is necessary for the process section + if(i==0) + out_count ++; + + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} From 3d2405323b0baa54e4796b47dc0c5e9d39e46dfc Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 29 May 2020 15:54:22 +0200 Subject: [PATCH 06/33] Add the downloading CenterNet weights and outputs for 3D Signed-off-by: Davide Sapienza --- tests/dla34_cnet3d/dla34_cnet3d.cpp | 2 +- tests/resnet101_cnet3d/resnet101_cnet3d.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dla34_cnet3d/dla34_cnet3d.cpp b/tests/dla34_cnet3d/dla34_cnet3d.cpp index ecd5693..9766b45 100644 --- a/tests/dla34_cnet3d/dla34_cnet3d.cpp +++ b/tests/dla34_cnet3d/dla34_cnet3d.cpp @@ -112,7 +112,7 @@ const char *output_bin[]={ int main() { - // downloadWeightsifDoNotExist(input_bin, "dla34_cnet3d", "https://cloud.hipert.unimore.it/s/KRZBbCQsKAtQwpZ/download"); + downloadWeightsifDoNotExist(input_bin, "dla34_cnet3d", "https://cloud.hipert.unimore.it/s/2MDyWGzQsTKMjmR/download"); // Network layout tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); diff --git a/tests/resnet101_cnet3d/resnet101_cnet3d.cpp b/tests/resnet101_cnet3d/resnet101_cnet3d.cpp index 0089a19..5d084be 100644 --- a/tests/resnet101_cnet3d/resnet101_cnet3d.cpp +++ b/tests/resnet101_cnet3d/resnet101_cnet3d.cpp @@ -194,7 +194,7 @@ const char *output_bin[]={ int main() { - // downloadWeightsifDoNotExist(input_bin, "resnet101_cnet3d", "https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download"); + downloadWeightsifDoNotExist(input_bin, "resnet101_cnet3d", "https://cloud.hipert.unimore.it/s/xH5oH9t5wdnktYf/download"); // Network layout tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); From 9f10c3f6e2ead551eb45f67538fd50e047f75056 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 7 Dec 2020 17:59:27 +0100 Subject: [PATCH 07/33] Add CenterTrack based on dla34. The implemented network refers to the nuScenes_3Dtracking.pth model. Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 + scripts/test_all_tests.sh | 39 +- .../dla34_cnet3d_track/dla34_cnet3d_track.cpp | 632 ++++++++++++++++++ 3 files changed, 655 insertions(+), 19 deletions(-) create mode 100644 tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3509d42..7ddc72a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,9 @@ target_link_libraries(test_resnet101_cnet3d tkDNN) add_executable(test_dla34_cnet3d tests/centernet/dla34_cnet3d/dla34_cnet3d.cpp) target_link_libraries(test_dla34_cnet3d tkDNN) +add_executable(test_dla34_cnet3d_track tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp) +target_link_libraries(test_dla34_cnet3d_track tkDNN) + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) diff --git a/scripts/test_all_tests.sh b/scripts/test_all_tests.sh index af04aff..09aa22a 100644 --- a/scripts/test_all_tests.sh +++ b/scripts/test_all_tests.sh @@ -72,27 +72,28 @@ do ./test_imuodom &>> $out_file print_output $? imuodom - test_net yolo4 - test_net yolo4x - test_net yolo4_berkeley - test_net yolo4tiny - test_net yolo3 - test_net yolo3_berkeley - test_net yolo3_coco4 - test_net yolo3_flir - test_net yolo3_512 - test_net yolo3tiny - test_net yolo3tiny_512 - test_net yolo2 - test_net yolo2_voc + # test_net yolo4 + # test_net yolo4x + # test_net yolo4_berkeley + # test_net yolo4tiny + # test_net yolo3 + # test_net yolo3_berkeley + # test_net yolo3_coco4 + # test_net yolo3_flir + # test_net yolo3_512 + # test_net yolo3tiny + # test_net yolo3tiny_512 + # test_net yolo2 + # test_net yolo2_voc #test_net yolo2tiny - test_net csresnext50-panet-spp + # test_net csresnext50-panet-spp #test_net csresnext50-panet-spp_berkeley - test_net resnet101_cnet - test_net dla34_cnet - test_net mobilenetv2ssd - test_net mobilenetv2ssd512 - test_net bdd-mobilenetv2ssd + # test_net resnet101_cnet + # test_net dla34_cnet + # test_net mobilenetv2ssd + # test_net mobilenetv2ssd512 + # test_net bdd-mobilenetv2ssd + test_net dla34_cnet3d_track done echo "If errors occured, check logfile $out_file" diff --git a/tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp b/tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp new file mode 100644 index 0000000..4829f16 --- /dev/null +++ b/tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp @@ -0,0 +1,632 @@ +#include +#include "tkdnn.h" +const char *input_bin = "dla34_cnet3d_track/debug/input_base-level0-0.bin"; +// const char *input_bin = "dla34_cnet3d_track/debug/input.bin"; +// const char *pre_img_bin = "dla34_cnet3d_track/debug/pre_imgages.bin"; +// const char *pre_hm_bin = "dla34_cnet3d_track/debug/pre_hms.bin"; +// //pre +// const char *pre_img_conv1_bin = "dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; +// const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; +// const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; + +const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; +const char *conv3_bin = "dla34_cnet3d_track/layers/base-level1-0.bin"; +// s - stage, t - tree +const char *s1_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level2-tree1-conv1.bin"; +const char *s1_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level2-tree1-conv2.bin"; +const char *s1_t1_project = "dla34_cnet3d_track/layers/base-level2-project-0.bin"; +const char *s1_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level2-tree2-conv1.bin"; +const char *s1_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level2-tree2-conv2.bin"; +const char *s1_root_conv1_bin = "dla34_cnet3d_track/layers/base-level2-root-conv.bin"; +const char *s2_t1_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree1-conv1.bin"; +const char *s2_t1_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree1-conv2.bin"; +const char *s2_t1_t1_project = "dla34_cnet3d_track/layers/base-level3-tree1-project-0.bin"; +const char *s2_t1_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree2-conv1.bin"; +const char *s2_t1_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree2-conv2.bin"; +const char *s2_t1_root_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree1-root-conv.bin"; +const char *s2_t2_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree1-conv1.bin"; +const char *s2_t2_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree1-conv2.bin"; +const char *s2_t2_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree2-conv1.bin"; +const char *s2_t2_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree2-conv2.bin"; +const char *s2_t2_root_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree2-root-conv.bin"; +const char *s3_t1_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree1-conv1.bin"; +const char *s3_t1_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree1-conv2.bin"; +const char *s3_t1_t1_project = "dla34_cnet3d_track/layers/base-level4-tree1-project-0.bin"; +const char *s3_t1_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree2-conv1.bin"; +const char *s3_t1_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree2-conv2.bin"; +const char *s3_t1_root_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree1-root-conv.bin"; +const char *s3_t2_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree1-conv1.bin"; +const char *s3_t2_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree1-conv2.bin"; +const char *s3_t2_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree2-conv1.bin"; +const char *s3_t2_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree2-conv2.bin"; +const char *s3_t2_root_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree2-root-conv.bin"; +const char *s4_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level5-tree1-conv1.bin"; +const char *s4_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level5-tree1-conv2.bin"; +const char *s4_t1_project = "dla34_cnet3d_track/layers/base-level5-project-0.bin"; +const char *s4_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level5-tree2-conv1.bin"; +const char *s4_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level5-tree2-conv2.bin"; +const char *s4_root_conv1_bin = "dla34_cnet3d_track/layers/base-level5-root-conv.bin"; + +//final +// const char *fc_bin = "dla34_cnet3d_track/layers/output.bin"; + +const char *ida_0_p_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-proj_1-conv.bin"; +const char *ida_0_p_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin"; +const char *ida_0_up_1_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-up_1.bin"; +const char *ida_0_n_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-node_1-conv.bin"; +const char *ida_0_n_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin"; + +const char *ida_1_p_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_1-conv.bin"; +const char *ida_1_p_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin"; +const char *ida_1_up_1_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-up_1.bin"; +const char *ida_1_n_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_1-conv.bin"; +const char *ida_1_n_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin"; +const char *ida_1_p_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_2-conv.bin"; +const char *ida_1_p_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin"; +const char *ida_1_up_2_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-up_2.bin"; +const char *ida_1_n_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_2-conv.bin"; +const char *ida_1_n_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin"; + +const char *ida_2_p_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_1-conv.bin"; +const char *ida_2_p_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin"; +const char *ida_2_up_1_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-up_1.bin"; +const char *ida_2_n_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_1-conv.bin"; +const char *ida_2_n_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin"; +const char *ida_2_p_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_2-conv.bin"; +const char *ida_2_p_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin"; +const char *ida_2_up_2_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-up_2.bin"; +const char *ida_2_n_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_2-conv.bin"; +const char *ida_2_n_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin"; +const char *ida_2_p_3_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_3-conv.bin"; +const char *ida_2_p_3_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin"; +const char *ida_2_up_3_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-up_3.bin"; +const char *ida_2_n_3_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_3-conv.bin"; +const char *ida_2_n_3_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin"; + +const char *ida_up_p_1_dcn_bin = "dla34_cnet3d_track/layers/ida_up-proj_1-conv.bin"; +const char *ida_up_p_1_conv_bin = "dla34_cnet3d_track/layers/ida_up-proj_1-conv-conv_offset_mask.bin"; +const char *ida_up_up_1_deconv_bin = "dla34_cnet3d_track/layers/ida_up-up_1.bin"; +const char *ida_up_n_1_dcn_bin = "dla34_cnet3d_track/layers/ida_up-node_1-conv.bin"; +const char *ida_up_n_1_conv_bin = "dla34_cnet3d_track/layers/ida_up-node_1-conv-conv_offset_mask.bin"; +const char *ida_up_p_2_dcn_bin = "dla34_cnet3d_track/layers/ida_up-proj_2-conv.bin"; +const char *ida_up_p_2_conv_bin = "dla34_cnet3d_track/layers/ida_up-proj_2-conv-conv_offset_mask.bin"; +const char *ida_up_up_2_deconv_bin = "dla34_cnet3d_track/layers/ida_up-up_2.bin"; +const char *ida_up_n_2_dcn_bin = "dla34_cnet3d_track/layers/ida_up-node_2-conv.bin"; +const char *ida_up_n_2_conv_bin = "dla34_cnet3d_track/layers/ida_up-node_2-conv-conv_offset_mask.bin"; + +const char *hm_conv1_bin = "dla34_cnet3d_track/layers/hm-0.bin"; +const char *hm_conv2_bin = "dla34_cnet3d_track/layers/hm-2.bin"; +const char *wh_conv1_bin = "dla34_cnet3d_track/layers/wh-0.bin"; +const char *wh_conv2_bin = "dla34_cnet3d_track/layers/wh-2.bin"; +const char *reg_conv1_bin = "dla34_cnet3d_track/layers/reg-0.bin"; +const char *reg_conv2_bin = "dla34_cnet3d_track/layers/reg-2.bin"; +const char *track_conv1_bin = "dla34_cnet3d_track/layers/tracking-0.bin"; +const char *track_conv2_bin = "dla34_cnet3d_track/layers/tracking-2.bin"; +const char *dep_conv1_bin = "dla34_cnet3d_track/layers/dep-0.bin"; +const char *dep_conv2_bin = "dla34_cnet3d_track/layers/dep-2.bin"; +const char *rot_conv1_bin = "dla34_cnet3d_track/layers/rot-0.bin"; +const char *rot_conv2_bin = "dla34_cnet3d_track/layers/rot-2.bin"; +const char *dim_conv1_bin = "dla34_cnet3d_track/layers/dim-0.bin"; +const char *dim_conv2_bin = "dla34_cnet3d_track/layers/dim-2.bin"; +const char *a_off_conv1_bin = "dla34_cnet3d_track/layers/amodel_offset-0.bin"; +const char *a_off_conv2_bin = "dla34_cnet3d_track/layers/amodel_offset-2.bin"; + +const char *output_bin[]={ +"dla34_cnet3d_track/debug/hm.bin", +"dla34_cnet3d_track/debug/wh.bin", +"dla34_cnet3d_track/debug/reg.bin", +"dla34_cnet3d_track/debug/tracking.bin", +"dla34_cnet3d_track/debug/dep.bin", +"dla34_cnet3d_track/debug/rot.bin", +"dla34_cnet3d_track/debug/dim.bin", +"dla34_cnet3d_track/debug/amodel_offset.bin"}; +// const char *output_bin = "dla34_cnet3d_track/debug/base-level0-2.bin"; +int main() +{ + + downloadWeightsifDoNotExist("dla34_cnet3d_track/debug/input.bin", "dla34_cnet3d_track", "https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download"); + + // Network layout + // tk::dnn::dataDim_t dim_in0(1, 3, 512, 512, 1); + // tk::dnn::dataDim_t dim_in1(1, 1, 512, 512, 1); + tk::dnn::dataDim_t dim_in0(1, 16, 512, 512, 1); + // dnnType *i0_d, *i1_d, *i2_d; + // dnnType *i0_h, *i1_h, *i2_h; + // checkCuda( cudaMalloc(&i0_d, dim_in0.tot()*sizeof(dnnType)) ); + // checkCuda( cudaMalloc(&i1_d, dim_in1.tot()*sizeof(dnnType)) ); + // checkCuda( cudaMalloc(&i2_d, dim_in0.tot()*sizeof(dnnType)) ); + + tk::dnn::Network net(dim_in0); + tk::dnn::Layer *last1, *last2, *last3, *last4; + tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2; + // tk::dnn::Layer *pre_img, *pre_hm; + + // //pre-img + // readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &i0_d); + // tk::dnn::Input *in_pre_img = new tk::dnn::Input(&net, dim_in0, i0_d); + // tk::dnn::Conv2d pre_img_conv1(&net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true); + // tk::dnn::Activation pre_img_relu(&net, CUDNN_ACTIVATION_RELU); + // pre_img = &pre_img_relu; + + // //pre-hm + // readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &i1_d); + // tk::dnn::Input *in_pre_hm = new tk::dnn::Input(&net, dim_in1, i1_d); + // tk::dnn::Conv2d pre_hm_conv1(&net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true); + // tk::dnn::Activation pre_hm_relu(&net, CUDNN_ACTIVATION_RELU); + // pre_hm = &pre_hm_relu; + + // // image input + // readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &i2_d); + // tk::dnn::Input *input_image = new tk::dnn::Input(&net, dim_in0, i2_d); + // tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + // tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU); + + // tk::dnn::Shortcut s0_input(&net, pre_img); + // tk::dnn::Shortcut s1_input(&net, pre_hm); + + tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true); + tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU); + base1 = &relu2; + + tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true); + tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU); + base2 = &relu3; + + // level 2 + // tree 1 + tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true); + tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true); + last2 = &s1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s1_t1_layers[1] = { base2 }; + tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1); + // downsample + tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + // project + tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true); + + tk::dnn::Shortcut s1_t1_s1(&net, last2); + tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s1_t1_relu; + // tree 2 + tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true); + tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true); + + tk::dnn::Shortcut s1_t2_s1(&net, last1); + tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2); + tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true); + tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base3 = &s1_root_relu; + + // level 3 + // tree 1 + // tree 1 + tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true); + tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true); + last2 = &s2_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s2_t1_t1_layers[1] = { base3 }; + tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s2_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true); + + tk::dnn::Shortcut s2_t1_t1_s1(&net, last2); + tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true); + tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t1_t2_s1(&net, last1); + tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2); + tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true); + tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_root_relu; + last3 = &s2_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true); + tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s2_t2_t1_s1(&net, last1); + tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true); + tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t2_t2_s1(&net, last1); + tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t2_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4); + tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true); + tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base4 = &s2_t2_root_relu; + + // level 4 + // tree 1 + // tree 1 + tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true); + tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true); + last2 = &s3_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s3_t1_t1_layers[1] = { base4 }; + tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s3_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true); + + tk::dnn::Shortcut s3_t1_t1_s1(&net, last2); + tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true); + tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t1_t2_s1(&net, last1); + tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t1_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2); + tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true); + tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_root_relu; + last3 = &s3_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true); + tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s3_t2_t1_s1(&net, last1); + tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true); + tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t2_t2_s1(&net, last1); + tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t2_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4); + tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true); + tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base5 = &s3_t2_root_relu; + + // level 5 + // tree 1 + tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true); + tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true); + last2 = &s4_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s4_t1_layers[1] = { base5 }; + tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1); + // downsample + tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s4_t1_maxpool1; + // project + tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true); + + tk::dnn::Shortcut s4_t1_s1(&net, last2); + tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s4_t1_relu; + + // tree 2 + tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true); + tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true); + + tk::dnn::Shortcut s4_t2_s1(&net, last1); + tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s4_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 }; + tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3); + tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true); + tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base6 = &s4_root_relu; + + //final + // tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE); + // tk::dnn::Dense fc(&net, 1000, fc_bin); + + //ida 0 + tk::dnn::Layer *route_ida0[1] = { base6 }; + tk::dnn::Route route_ida0_0(&net, route_ida0, 1); + + tk::dnn::DeformConv2d ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true); + tk::dnn::Activation ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256); + tk::dnn::Shortcut ida_0_shortcut(&net, base5); + tk::dnn::DeformConv2d ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true); + tk::dnn::Activation ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida1 = &ida_0_n_1_relu; + + //ida1-1 + tk::dnn::Layer *route_ida1_layers_1[1] = { base5 }; + tk::dnn::Route route_ida1_1(&net, route_ida1_layers_1, 1); + + tk::dnn::DeformConv2d ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true); + tk::dnn::Activation ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut1(&net, base4); + tk::dnn::DeformConv2d ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true); + tk::dnn::Activation ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_1 = &ida_1_n_1_relu; + + //ida1-2 + tk::dnn::Layer *route_ida1_layers_2[1] = { ida1 }; + tk::dnn::Route route_ida1_2(&net, route_ida1_layers_2, 1); + + tk::dnn::DeformConv2d ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true); + tk::dnn::Activation ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut2(&net, ida2_1); + tk::dnn::DeformConv2d ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true); + tk::dnn::Activation ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_2 = &ida_1_n_2_relu; + + //ida2-1 + tk::dnn::Layer *route_ida2_layers_1[1] = { base4 }; + tk::dnn::Route route_ida2_1(&net, route_ida2_layers_1, 1); + + tk::dnn::DeformConv2d ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true); + tk::dnn::Activation ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut1(&net, base3); + tk::dnn::DeformConv2d ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true); + tk::dnn::Activation ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_1 = &ida_2_n_1_relu; + + //ida2-2 + tk::dnn::Layer *route_ida2_layers_2[1] = { ida2_1 }; + tk::dnn::Route route_ida2_2(&net, route_ida2_layers_2, 1); + + tk::dnn::DeformConv2d ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true); + tk::dnn::Activation ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut2(&net, ida3_1); + tk::dnn::DeformConv2d ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true); + tk::dnn::Activation ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_2 = &ida_2_n_2_relu; + + //ida2-3 + tk::dnn::Layer *route_ida2_layers_3[1] = { ida2_2 }; + tk::dnn::Route route_ida2_3(&net, route_ida2_layers_3, 1); + + tk::dnn::DeformConv2d ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true); + tk::dnn::Activation ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut3(&net, ida3_2); + tk::dnn::DeformConv2d ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true); + tk::dnn::Activation ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_3 = &ida_2_n_3_relu; + + //idaup-1 + tk::dnn::Layer *route_idaup_layers_1[1] = { ida2_2 }; + tk::dnn::Route route_idaup_1(&net, route_idaup_layers_1, 1); + + tk::dnn::DeformConv2d idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true); + tk::dnn::Activation idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut1(&net, ida3_3); + tk::dnn::DeformConv2d idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true); + tk::dnn::Activation idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_1 = &idaup_n_1_relu; + + //idaup-2 + tk::dnn::Layer *route_idaup_layers_2[1] = { ida1 }; + tk::dnn::Route route_idaup_2(&net, route_idaup_layers_2, 1); + + tk::dnn::DeformConv2d idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true); + tk::dnn::Activation idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut2(&net, idaup_1); + tk::dnn::DeformConv2d idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true); + tk::dnn::Activation idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_2 = &idaup_n_2_relu; + + tk::dnn::Layer *route_1_0_layers[1] = { idaup_2 }; + + // hm + tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false); + tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 10, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false); + hm->setFinal(); + + int kernel = 3; + int pad = (kernel - 1)/2; + tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX); + hmax->setFinal(); + + // wh + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false); + tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false); + wh->setFinal(); + + // reg + tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false); + tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false); + reg->setFinal(); + + // tracking + tk::dnn::Route *route_3_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *track_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, track_conv1_bin, false); + tk::dnn::Activation *track_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *track = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, track_conv2_bin, false); + track->setFinal(); + + // dep + tk::dnn::Route *route_4_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dep_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dep_conv1_bin, false); + tk::dnn::Activation *dep_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dep = new tk::dnn::Conv2d(&net, 1, 1, 1, 1, 1, 0, 0, dep_conv2_bin, false); + dep->setFinal(); + + // rot + tk::dnn::Route *route_5_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *rot_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, rot_conv1_bin, false); + tk::dnn::Activation *rot_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *rot = new tk::dnn::Conv2d(&net, 8, 1, 1, 1, 1, 0, 0, rot_conv2_bin, false); + rot->setFinal(); + + // dim + tk::dnn::Route *route_6_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dim_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dim_conv1_bin, false); + tk::dnn::Activation *dim_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dim_ = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, dim_conv2_bin, false); + dim_->setFinal(); + + // amodel_offset + tk::dnn::Route *route_7_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *a_off_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, a_off_conv1_bin, false); + tk::dnn::Activation *a_off_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *a_off = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, a_off_conv2_bin, false); + a_off->setFinal(); + + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim_in0.tot(), &input_h, &data); + //printDeviceVector(64, data, true); + + //print network model + net.print(); + + //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet3d_track")); + + tk::dnn::dataDim_t dim1 = dim_in0; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TKDNN_TSTART + // tk::dnn::dataDim_t dim_aus; + // net.infer(dim_aus, nullptr); + net.infer(dim1, data); + TKDNN_TSTOP + dim1.print(); + } + + tk::dnn::dataDim_t dim2 = dim_in0; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TKDNN_TSTART + netRT.infer(dim2, data); + TKDNN_TSTOP + dim2.print(); + } + // dnnType *out, *out_h; + // int odim = net.layers[net.num_layers-1]->output_dim.tot(); + // readBinaryFile(output_bin, odim, &out_h, &out); + // dnnType *cudnn_out; + // cudnn_out = net.layers[net.num_layers-1]->dstData; + // std::cout<<"CUDNN vs correct"; + // checkResult(odim, cudnn_out, out); + + + tk::dnn::Layer *outs[8] = { hm, wh, reg, track, dep, rot, dim_, a_off}; + int out_count = 1; + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for(int i=0; i<8; i++) { + printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i+out_count]; + // there is the maxpool. It isn't an output but it is necessary for the process section + if(i==0) + out_count ++; + + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} From 48ecebe6dd187b440f1f6668de4d5634ddba9357 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 7 Dec 2020 18:45:29 +0100 Subject: [PATCH 08/33] Add CenterTrack pre, post, visualization and demo. Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 5 + include/tkDNN/CenternetDetection3DTrack.h | 176 +++++ src/CenternetDetection3DTrack.cpp | 862 ++++++++++++++++++++++ 3 files changed, 1043 insertions(+) create mode 100644 include/tkDNN/CenternetDetection3DTrack.h create mode 100644 src/CenternetDetection3DTrack.cpp diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index e838395..1a93206 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -5,6 +5,7 @@ #include #include "CenternetDetection3D.h" +#include "CenternetDetection3DTrack.h" bool gRun; bool SAVE_RESULT = false; @@ -34,6 +35,7 @@ int main(int argc, char *argv[]) { n_classes = atoi(argv[4]); tk::dnn::CenternetDetection3D cnet; + tk::dnn::CenternetDetection3DTrack ctrack; tk::dnn::DetectionNN3D *detNN; @@ -42,6 +44,9 @@ int main(int argc, char *argv[]) { case 'c': detNN = &cnet; break; + case 't': + detNN = &ctrack; + break; default: FatalError("Network type not allowed (3rd parameter)\n"); } diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h new file mode 100644 index 0000000..5149d70 --- /dev/null +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -0,0 +1,176 @@ +#ifndef CENTERNETDETECTION3DTRACK_H +#define CENTERNETDETECTION3DTRACK_H + +#include "kernels.h" +#include "utils.h" +#include "tkdnn.h" +#include +#include "opencv2/opencv.hpp" +#include +#include +#include // std::iota +#include // std::sort + +#include "DetectionNN3D.h" + +#include "kernelsThrust.h" + + +namespace tk { namespace dnn { + +struct detectionRes +{ + float score; + int cl; + cv::Mat ct, tr, bb0, bb1; + float dep; + float dim[3]; + float alpha; + float x,y,z; + float rot_y; + detectionRes() : ct(cv::Mat(cv::Size(1,2), CV_32F)), + tr(cv::Mat(cv::Size(1,2), CV_32F)), + bb0(cv::Mat(cv::Size(1,2), CV_32F)), + bb1(cv::Mat(cv::Size(1,2), CV_32F)) { } + ~detectionRes() { + ct.release(); + tr.release(); + bb0.release(); + bb1.release(); + } +}; + +struct trackingRes +{ + struct detectionRes det_res; + int tracking_id; + int age; + int active; + int color; +}; + +class CenternetDetection3DTrack : public DetectionNN3D +{ +private: + tk::dnn::dataDim_t dim; + tk::dnn::dataDim_t dim2; + tk::dnn::dataDim_t dim_hm; + tk::dnn::dataDim_t dim_wh; + tk::dnn::dataDim_t dim_reg; + tk::dnn::dataDim_t dim_track; + tk::dnn::dataDim_t dim_dep; + tk::dnn::dataDim_t dim_rot; + tk::dnn::dataDim_t dim_dim; + tk::dnn::dataDim_t dim_amodel_offset; + + /* preprocessing */ + #ifdef OPENCV_CUDACONTRIB + float *mean_d; + float *stddev_d; + #else + cv::Vec mean; + cv::Vec stddev; + dnnType *input; + #endif + float *d_ptrs; + + cv::Mat src; + cv::Mat dst; + cv::Mat dst2; + cv::Mat trans, trans2, trans_out; + + /* pre inf */ + bool iter0; + dnnType *input_pre_inf_d; + bool test_pre_inf = true; + dnnType *img_d, *hm_d; + tk::dnn::dataDim_t dim_in0; + tk::dnn::dataDim_t dim_in1; + dnnType *out_d; + + + /* postprocessing */ + int K = 100; + int width = 128;//56; // TODO + + // pointer used in the kernels + float *src_out; + int *ids_out; + + float *topk_scores; + int *topk_inds_; + float *topk_ys_; + float *topk_xs_; + int *ids_d, *ids_; + + float *ones; + + float *scores, *scores_d; + int *clses, *clses_d; + int *topk_inds_d; + float *topk_ys_d; + float *topk_xs_d; + int *inttopk_xs_d, *inttopk_ys_d; + + float *bbx0, *bby0, *bbx1, *bby1; + float *bbx0_d, *bby0_d, *bbx1_d, *bby1_d; + + int *intxs, *intys; + + float *track, *dep, *rot, *dim_, *wh, *amodel_offset; + float *track_d, *dep_d, *rot_d, *dim_d, *wh_d, *amodel_offset_d; + + float *target_coords; + + /* visualization */ + cv::Mat r; + cv::Mat calibs; + cv::Mat corners, pts3DHomo; + + std::vector> face_id; + cv::Scalar tr_colors[256]; + bool view2d = false; + + //processing + struct threshold op; + float out_thresh = 0.1; + float new_thresh = 0.3; + float vis_thresh = 0.3; + float peakThreshold = 0.2; + float centerThreshold = 0.3; //default 0.5 + + + //detections + std::vector det_res; + int count_det; + //tracks + std::vector tr_res; + int count_tr; + int track_id=0; + + + bool init_preprocessing(); + bool init_pre_inf(); + bool init_postprocessing(); + bool init_visualization(const int n_classes); + void pre_inf(); + void _get_additional_inputs(); + cv::Mat transform_preds_with_trans(float x1, float x2); + void tracking(); + +public: + tk::dnn::Network *pre_phase_net = nullptr; + CenternetDetection3DTrack() {}; + ~CenternetDetection3DTrack() {}; + bool init(const std::string& tensor_path, const int n_classes=3); + void preprocess(cv::Mat &frame); + void postprocess(); + cv::Mat draw(cv::Mat &frame); +}; + + +} // namespace dnn +} // namespace tk + + +#endif /*CENTERNETDETECTION3DTRACK_H*/ \ No newline at end of file diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp new file mode 100644 index 0000000..35488f7 --- /dev/null +++ b/src/CenternetDetection3DTrack.cpp @@ -0,0 +1,862 @@ +#include "CenternetDetection3DTrack.h" + + +namespace tk { namespace dnn { + +bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes){ + std::cout<<(tensor_path).c_str()<<"\n"; + netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); + + dim = netRT->input_dim; + dim.c = 3; + + init_preprocessing(); + init_pre_inf(); + init_postprocessing(); + init_visualization(n_classes); + + count_tr = 0; +} + +bool CenternetDetection3DTrack::init_preprocessing(){ + //image transformation + src = cv::Mat(cv::Size(2,3), CV_32F); + dst = cv::Mat(cv::Size(2,3), CV_32F); + dst2 = cv::Mat(cv::Size(2,3), CV_32F); + trans = cv::Mat(cv::Size(3,2), CV_32F); + trans2 = cv::Mat(cv::Size(3,2), CV_32F); + trans_out = cv::Mat(cv::Size(3,2), CV_32F); + + dst2.at(0,0)=width * 0.5; + dst2.at(0,1)=width * 0.5; + dst2.at(1,0)=width * 0.5; + dst2.at(1,1)=width * 0.5 + width * -0.5; + + dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); + dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); + + +#ifdef OPENCV_CUDACONTRIB + + checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) ); + checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) ); + float mean[3] = {0.40789655, 0.44719303, 0.47026116}; + float stddev[3] = {0.2886383, 0.27408165, 0.27809834}; + + checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); + checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); +#else + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot())); + mean << 0.40789655, 0.44719303, 0.47026116; + stddev << 0.2886383, 0.27408165, 0.27809834; + +#endif + + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); + checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); +} + +bool CenternetDetection3DTrack::init_pre_inf(){ + // initial steps: the first part of the network + const char *pre_img_conv1_bin = "/home/davide/Projects/repos/tkDNN/build/dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; + const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; + const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; + const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; + dim_in0 = tk::dnn::dataDim_t(1, 3, 512, 512, 1); + dim_in1 = tk::dnn::dataDim_t(1, 1, 512, 512, 1); + + + checkCuda( cudaMalloc(&out_d, netRT->input_dim.tot()*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&img_d, dim_in0.tot()*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&hm_d, dim_in1.tot()*sizeof(dnnType)) ); + // init to zeros hm + dnnType *hm_h; + checkCuda( cudaMallocHost(&hm_h, 1 * dim.h * dim.w*sizeof(dnnType)) ); + for(int i=0; i<1 * dim.h * dim.w; i++) + hm_h[i]=0.0f; + checkCuda( cudaMemcpy(hm_d, hm_h, 1 * dim.h * dim.w * sizeof(dnnType), cudaMemcpyHostToDevice) ); + checkCuda( cudaFreeHost(hm_h) ); + dnnType *i0_h, *i1_h, *i2_h; + // dnnType *i0_d, *i1_d, *i2_d; + + // const char *input_bin = "dla34_cnet3d_track/debug/input.bin"; + // const char *pre_img_bin = "dla34_cnet3d_track/debug/pre_imgages.bin"; + // const char *pre_hm_bin = "dla34_cnet3d_track/debug/pre_hms.bin"; + // readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &img_d); + // readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &hm_d); + // readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &input_pre_inf_d); + + pre_phase_net = new tk::dnn::Network(dim_in0); + //pre-img + tk::dnn::Input *in_pre_img = new tk::dnn::Input(pre_phase_net, dim_in0, img_d); + tk::dnn::Conv2d *pre_img_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true); + tk::dnn::Activation *pre_img_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + //pre-hm + tk::dnn::Input *in_pre_hm = new tk::dnn::Input(pre_phase_net, dim_in1, hm_d); + tk::dnn::Conv2d *pre_hm_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true); + tk::dnn::Activation *pre_hm_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + // image input + tk::dnn::Input *input_image = new tk::dnn::Input(pre_phase_net, dim_in0, input_pre_inf_d); + tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation *relu1 = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Shortcut *s0_input = new tk::dnn::Shortcut(pre_phase_net, pre_img_relu); + tk::dnn::Shortcut *s1_input = new tk::dnn::Shortcut(pre_phase_net, pre_hm_relu); + // output data + out_d = s1_input->dstData; + //print network model + pre_phase_net->print(); + + iter0=true; // in the first iteration the last input is equal to the current input. + return true; +} + +bool CenternetDetection3DTrack::init_postprocessing(){ + srand(0); //seed = 0 for random colors + + dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1); + dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_track = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1); + dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1); + dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1); + dim_amodel_offset = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + + checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) ); + checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); + checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); + for(int i =0; i(0,0) = 633.0; + calibs.at(0,1) = 0.0; + calibs.at(0,2) = 0.0; //w/2 + calibs.at(0,3) = 0.0; + calibs.at(1,0) = 0.0; + calibs.at(1,1) = 633.0; + calibs.at(1,2) = 0.0; //h/2 + calibs.at(1,3) = 0.0; + calibs.at(2,0) = 0.0; + calibs.at(2,1) = 0.0; + calibs.at(2,2) = 1.0; + calibs.at(2,3) = 0.0; + + // Alloc array used in the kernel + checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); +} + +bool CenternetDetection3DTrack::init_visualization(const int n_classes){ + classes = n_classes; + // const char *kitti_class_name[] = { + // "person", "car", "bicycle"}; + // classesNames = std::vector(kitti_class_name, std::end( kitti_class_name)); + + const char *class_name[] = {"car", "truck", "bus", "trailer", "construction_vehicle", "pedestrian", + "motorcycle", "bicycle", "traffic_cone", "barrier"}; + classesNames = std::vector(class_name, std::end( class_name)); + + // const char *coco_class_name[] = { + // "person", "bicycle", "car", "motorcycle", "airplane", + // "bus", "train", "truck", "boat", "traffic light", "fire hydrant", + // "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", + // "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", + // "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", + // "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", + // "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", + // "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", + // "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", + // "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", + // "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", + // "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", + // "scissors", "teddy bear", "hair drier", "toothbrush" + // }; + // classesNames = std::vector(coco_class_name, std::end( coco_class_name)); + + for(int c=0; c(0,1) = 0.0; + r.at(1,0) = 0.0; + r.at(1,1) = 1.0; + r.at(1,2) = 0.0; + r.at(2,1) = 0.0; + + corners = cv::Mat(cv::Size(8,3), CV_32F); + corners.at(1,0) = 0.0; + corners.at(1,1) = 0.0; + corners.at(1,2) = 0.0; + corners.at(1,3) = 0.0; + + pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F); + pts3DHomo.at(3,0) = 1.0; + pts3DHomo.at(3,1) = 1.0; + pts3DHomo.at(3,2) = 1.0; + pts3DHomo.at(3,3) = 1.0; + pts3DHomo.at(3,4) = 1.0; + pts3DHomo.at(3,5) = 1.0; + pts3DHomo.at(3,6) = 1.0; + pts3DHomo.at(3,7) = 1.0; + + face_id.push_back({0,1,5,4}); + face_id.push_back({1,2,6, 5}); + face_id.push_back({2,3,7,6}); + face_id.push_back({3,0,4,7}); + // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); +} + +void CenternetDetection3DTrack::_get_additional_inputs(){ + //None no additional input +} + +void CenternetDetection3DTrack::pre_inf(){ + TKDNN_TSTART + tk::dnn::dataDim_t dim_aus; + pre_phase_net->infer(dim_aus, nullptr); + TKDNN_TSTOP + checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaMemcpy(input_d, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaDeviceSynchronize() ); +} + +void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ + // -----------------------------------pre-process ------------------------------------------ + + cv::Size sz = originalSize; + cv::Size sz_old; + float scale = 1.0; + float new_height = sz.height * scale; + float new_width = sz.width * scale; + if(sz.height != sz_old.height && sz.width != sz_old.width){ + calibs.at(0,2) = new_width / 2.0f; + calibs.at(1,2) = new_height /2.0f; + float c[] = {new_width / 2.0f, new_height /2.0f}; + float s[] = {dim.w, dim.h}; + // float s = new_width >= new_height ? new_width : new_height; + // ----------- get_affine_transform + // rot_rad = pi * 0 / 100 --> 0 + dim.print(); + src.at(0,0)=c[0]; + src.at(0,1)=c[1]; + src.at(1,0)=c[0]; + src.at(1,1)=c[1] + s[0] * -0.5; + dst.at(0,0)=dim.w * 0.5; + dst.at(0,1)=dim.h * 0.5; + dst.at(1,0)=dim.w * 0.5; + dst.at(1,1)=dim.h * 0.5 + dim.w * -0.5; + + src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); + src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); + dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); + dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); + + + trans = cv::getAffineTransform( src, dst ); + trans2 = cv::getAffineTransform( dst2, src ); + trans2.convertTo(trans_out, CV_32F); + } + sz_old = sz; +#ifdef OPENCV_CUDACONTRIB + std::cout<<"OPENCV CPMTROB\n"; + cv::cuda::GpuMat im_Orig; + cv::cuda::GpuMat imageF1_d, imageF2_d; + + im_Orig = cv::cuda::GpuMat(frame); + // cv::cuda::resize (im_Orig, imageF1_d, cv::Size(new_width, new_height)); + imageF1_d = im_Orig; + checkCuda( cudaDeviceSynchronize() ); + + sz = imageF1_d.size(); + + cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR ); + checkCuda( cudaDeviceSynchronize() ); + + imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0); + checkCuda( cudaDeviceSynchronize() ); + + dim2 = dim; + cv::cuda::GpuMat bgr[3]; + cv::cuda::split(imageF1_d,bgr);//split source + + for(int i=0; i(0,0) = x1; + target_coords.at(0,1) = x2; + target_coords.at(0,2) = 1.0; + return trans_out * target_coords; +} + +void CenternetDetection3DTrack::tracking(){ + + float item_size[count_det]; + int item_cl[count_det]; + float dets[2*count_det]; + for(int i=0; i(0,0) - det_res[i].bb0.at(0,0)) * + (det_res[i].bb1.at(0,1) - det_res[i].bb0.at(0,1)); + item_cl[i] = det_res[i].cl; + dets[i*2] = det_res[i].ct.at(0,0); + dets[i*2+1] = det_res[i].ct.at(0,1); + } + + float track_size[count_tr]; + int track_cl[count_tr]; + float tracks[2*count_tr]; + for(int i=0; i(0,0) - tr_res[i].det_res.bb0.at(0,0)) * + (tr_res[i].det_res.bb1.at(0,1) - tr_res[i].det_res.bb0.at(0,1)); + track_cl[i] = tr_res[i].det_res.cl; + tracks[i*2] = tr_res[i].det_res.ct.at(0,0); + tracks[i*2+1] = tr_res[i].det_res.ct.at(0,1); + } + float dist[count_tr*count_det]; + bool invalid; + for(int i=0; i track_size[i] || dist[j*count_tr+i] > item_size[j] || item_cl[j] != track_cl[i]; + dist[j*count_tr+i] = dist[j*count_tr+i] + invalid * (1 << 18); + } + } + int matched_indices[2*count_tr]; + float min_tr; + int min_idtr=-1; + for(int i=0; i new_tr_res; + int id_new_tr=0; + for(int i=0; i new_thresh) { + count_tr_ ++; + struct trackingRes new_tr_res_; + new_tr_res_.det_res.score = det_res[i].score; + new_tr_res_.det_res.cl = det_res[i].cl; + new_tr_res_.det_res.ct = det_res[i].ct; + new_tr_res_.det_res.tr = det_res[i].tr; + new_tr_res_.det_res.bb0 = det_res[i].bb0; + new_tr_res_.det_res.bb1 = det_res[i].bb1; + new_tr_res_.det_res.dep = det_res[i].dep; + new_tr_res_.det_res.dim[0] = det_res[i].dim[0]; + new_tr_res_.det_res.dim[1] = det_res[i].dim[1]; + new_tr_res_.det_res.dim[2] = det_res[i].dim[2]; + new_tr_res_.det_res.alpha = det_res[i].alpha; + new_tr_res_.det_res.x = det_res[i].x; + new_tr_res_.det_res.y = det_res[i].y; + new_tr_res_.det_res.z = det_res[i].z; + new_tr_res_.det_res.rot_y = det_res[i].rot_y; + new_tr_res_.tracking_id = track_id++; + new_tr_res_.age = 1; + new_tr_res_.active = 1; + new_tr_res_.color = rand() % 256; + tr_res.push_back(new_tr_res_); + } + } + count_tr = count_tr_; + + if(track_id==1000) + track_id=0; + det_res.clear(); + +} + +void CenternetDetection3DTrack::postprocess(){ + dnnType *rt_out[9]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]; + rt_out[1] = (dnnType *)netRT->buffersRT[2]; + rt_out[2] = (dnnType *)netRT->buffersRT[3]; + rt_out[3] = (dnnType *)netRT->buffersRT[4]; + rt_out[4] = (dnnType *)netRT->buffersRT[5]; + rt_out[5] = (dnnType *)netRT->buffersRT[6]; + rt_out[6] = (dnnType *)netRT->buffersRT[7]; + rt_out[7] = (dnnType *)netRT->buffersRT[8]; + rt_out[8] = (dnnType *)netRT->buffersRT[9]; + + // ------------------------------------ process -------------------------------------------- + + activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot()); + checkCuda( cudaDeviceSynchronize() ); + + // output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1. + activationSIGMOIDForward(rt_out[5], rt_out[5], dim_dep.tot()); + checkCuda( cudaDeviceSynchronize() ); + transformDep(ones, ones + dim_dep.tot(), rt_out[5], rt_out[5] + dim_dep.tot()); + checkCuda( cudaDeviceSynchronize() ); + + // nms + subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op); + + // ----------- nms end + // ----------- topk + + if(K > dim_hm.h * dim_hm.w){ + printf ("Error topk (K is too large)\n"); + return; + } + + checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) ); + + sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d); + checkCuda( cudaDeviceSynchronize() ); + + topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d); + checkCuda( cudaDeviceSynchronize() ); + + checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) ); + + topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d); + checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); + + checkCuda( cudaMemcpy(intxs, inttopk_xs_d, K * sizeof(int), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(intys, inttopk_ys_d, K * sizeof(int), cudaMemcpyDeviceToHost) ); + + checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); + + // ----------- topk end + + topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out); + checkCuda( cudaDeviceSynchronize() ); + + bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out); + checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(bby0, bby0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(bbx1, bbx1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + + //regression heads + // ['tracking', 'dep', 'rot', 'dim', 'amodel_offset', + // 'nuscenes_att', 'velocity'] + getRecordsFromTopKId(topk_inds_d, K, dim_track.c, dim_track.h * dim_track.w, rt_out[4], track_d, ids_out); + checkCuda( cudaMemcpy(track, track_d, K * dim_track.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[5], dep_d, ids_out); + checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[6], rot_d, ids_out); + checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[7], dim_d, ids_out); + checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_amodel_offset.c, dim_amodel_offset.h * dim_amodel_offset.w, rt_out[8], amodel_offset_d, ids_out); + checkCuda( cudaMemcpy(amodel_offset, amodel_offset_d, K * dim_amodel_offset.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + // ---------------------------------- post-process ----------------------------------------- + + count_det = 0; + det_res.clear(); + for(int i = 0; i(2,3); + new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs.at(0,3) - calibs.at(0,2) * new_det_res.z) / calibs.at(0,0); + new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs.at(1,3) - calibs.at(1,2) * new_det_res.z) / calibs.at(1,1) + (dim_[i] / 2); + + // alpha2rot_y + // idx = rot[:, 1] > rot[:, 5] + // alpha1 = np.arctan2(rot[:, 2], rot[:, 3]) + (-0.5 * np.pi) + // alpha2 = np.arctan2(rot[:, 6], rot[:, 7]) + ( 0.5 * np.pi) + // return alpha1 * idx + alpha2 * (1 - idx) + if(rot[1*K + i] > rot[5*K + i]) + new_det_res.alpha = std::atan2(rot[2*K + i], rot[3*K + i]) -0.5 * M_PI; + else + new_det_res.alpha = std::atan2(rot[6*K + i], rot[7*K + i]) +0.5 * M_PI; + new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at(0,0) - calibs.at(0,2), calibs.at(0,0))); + new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest + det_res.push_back(new_det_res); + + } + // track step + tracking(); +} + +cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) { + + float sc; + int id; + std::string txt; + int baseline = 0; + float font_scale = 0.8; + int thickness = 2; + for(int i=0; i vis_thresh){// && tr_res[i].active!=0) { + if(view2d) { + + + cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), tr_res[i].det_res.bb0.at(0,1)), + cv::Point(tr_res[i].det_res.bb1.at(0,0), tr_res[i].det_res.bb1.at(0,1)), tr_colors[tr_res[i].color], thickness); + cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), + tr_res[i].det_res.bb0.at(0,1) - text_size.height - thickness), + cv::Point(tr_res[i].det_res.bb0.at(0,0) + text_size.width, + tr_res[i].det_res.bb0.at(0,1)), tr_colors[tr_res[i].color], -1); + + cv::putText(frame, txt, cv::Point(tr_res[i].det_res.bb0.at(0,0), + tr_res[i].det_res.bb0.at(0,1) - thickness -1), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); + + cv::arrowedLine(frame, cv::Point((int)tr_res[i].det_res.ct.at(0,0), + (int)tr_res[i].det_res.ct.at(0,1)), + cv::Point((int)(tr_res[i].det_res.ct.at(0,0) + tr_res[i].det_res.tr.at(0,0)), + (int)(tr_res[i].det_res.ct.at(0,1) + tr_res[i].det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); + } + //3d + if(!view2d && tr_res[i].det_res.z > 1){ + r.at(0,0) = std::cos(tr_res[i].det_res.rot_y); + r.at(0,2) = std::sin(tr_res[i].det_res.rot_y); + r.at(2,0) = -std::sin(tr_res[i].det_res.rot_y); + r.at(2,2) = std::cos(tr_res[i].det_res.rot_y); + + corners.at(0,0) = tr_res[i].det_res.dim[2]/2; + corners.at(0,1) = tr_res[i].det_res.dim[2]/2; + corners.at(0,2) = -tr_res[i].det_res.dim[2]/2; + corners.at(0,3) = -tr_res[i].det_res.dim[2]/2; + corners.at(0,4) = tr_res[i].det_res.dim[2]/2; + corners.at(0,5) = tr_res[i].det_res.dim[2]/2; + corners.at(0,6) = -tr_res[i].det_res.dim[2]/2; + corners.at(0,7) = -tr_res[i].det_res.dim[2]/2; + + corners.at(1,4) = -tr_res[i].det_res.dim[0]; + corners.at(1,5) = -tr_res[i].det_res.dim[0]; + corners.at(1,6) = -tr_res[i].det_res.dim[0]; + corners.at(1,7) = -tr_res[i].det_res.dim[0]; + + corners.at(2,0) = tr_res[i].det_res.dim[1]/2; + corners.at(2,1) = -tr_res[i].det_res.dim[1]/2; + corners.at(2,2) = -tr_res[i].det_res.dim[1]/2; + corners.at(2,3) = tr_res[i].det_res.dim[1]/2; + corners.at(2,4) = tr_res[i].det_res.dim[1]/2; + corners.at(2,5) = -tr_res[i].det_res.dim[1]/2; + corners.at(2,6) = -tr_res[i].det_res.dim[1]/2; + corners.at(2,7) = tr_res[i].det_res.dim[1]/2; + + cv::Mat aus = r * corners; + + for(int k=0; k<8; k++) { + aus.at(0,k) += tr_res[i].det_res.x; + aus.at(1,k) += tr_res[i].det_res.y; + aus.at(2,k) += tr_res[i].det_res.z; + } + + // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); + for(int k1=0; k1<3; k1++) { + for(int k2=0; k2<8; k2++) + pts3DHomo.at(k1,k2) = aus.at(k1,k2); + } + + aus.release(); + aus = calibs * pts3DHomo; + std::vector res_corners; + for(int k=0; k<8; k++) { + res_corners.push_back(aus.at(0,k) / aus.at(2,k)); + res_corners.push_back(aus.at(1,k) / aus.at(2,k)); + } + aus.release(); + for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(j) * 2), + res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point(res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + tr_colors[tr_res[i].color], 2); + if(ind_f == 0) { + cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(0) * 2), + res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point(res_corners.at(face_id.at(ind_f).at(2) * 2), + res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2); + cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(1) * 2), + res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point(res_corners.at(face_id.at(ind_f).at(3) * 2), + res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2); + } + } + } + float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; + for(int k=0; k<8; k++) { + if(res_corners[2*k]bb1) + bb1=res_corners[2*k]; + if(res_corners[2*k+1]bb3) + bb3=res_corners[2*k+1]; + + } + cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), + tr_colors[tr_res[i].color], thickness); + cv::rectangle(frame, cv::Point(bb0, bb2 - text_size.height - thickness), + cv::Point(bb0 + text_size.width, bb2), tr_colors[tr_res[i].color], -1); + + cv::putText(frame, txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, + font_scale, cv::Scalar(255, 255, 255), 1); + + cv::arrowedLine(frame, cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), + cv::Point((int)((bb0 + bb1)/2 + tr_res[i].det_res.tr.at(0,0)), + (int)((bb2 + bb3)/2 + tr_res[i].det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); + } + } + + } + return frame; +} + +}} + + From 4543df853390c5512653419905b3906733b3b378 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 7 Dec 2020 18:56:07 +0100 Subject: [PATCH 09/33] Update readme. Signed-off-by: Davide Sapienza --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 234e66b..3e4ceac 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,16 @@ cd pytorch-ssd conda env create -f env_mobv2ssd.yml python run_ssd_live_demo.py mb2-ssd-lite ``` +### 5)Export weights for CenterTrack +To get the weights needed to run CenterTrack tests use [this](https://github.com/sapienzadavide/CenterTrack.git) fork of the original CenterTrack. +``` +git clone https://github.com/sapienzadavide/CenterTrack.git +``` +* follow the instruction in the README.md and INSTALL.md + +``` +python demo.py tracking,ddd --load_model ../models/nuScenes_3Dtracking.pth --dataset nuscenes --pre_hm --track_thresh 0.1 --demo /path/to/image/or/folder/or/video/or/webcam --test_focal_length 633 --exp_wo --exp_wo_dim 512 --input_h 512 --input_w 512 +``` ## Darknet Parser tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*: @@ -246,6 +256,15 @@ The demo3D program takes the same parameters of the demo program: ./demo ``` +#### Run the 3D OD-tracking demo + +To run the 3D object detection & tracking demo follow these steps (example with CenterTrack based on DLA34): +``` +rm dla34_cnet3d_track_fp32.rt # be sure to delete(or move) old tensorRT files +./test_dla34_cnet3d_track # run the yolo test (is slow) +./demo3D dla34_cnet3d_track_fp32.rt ../demo/yolo_test.mp4 t +``` + ### FP16 inference To run the an object detection demo with FP16 inference follow these steps (example with yolov3): From dbc052865c0c2c74994786bd62938f44d4b5a674 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 9 Dec 2020 18:04:42 +0100 Subject: [PATCH 10/33] Fix a wrong path in CenternetDetection3DTrack.cpp Signed-off-by: Davide Sapienza --- src/CenternetDetection3DTrack.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 35488f7..6b34571 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -59,7 +59,7 @@ bool CenternetDetection3DTrack::init_preprocessing(){ bool CenternetDetection3DTrack::init_pre_inf(){ // initial steps: the first part of the network - const char *pre_img_conv1_bin = "/home/davide/Projects/repos/tkDNN/build/dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; + const char *pre_img_conv1_bin = "dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; From 9e1d7b3bb42f870b417a3229bf495e575377aed7 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 9 Dec 2020 18:05:54 +0100 Subject: [PATCH 11/33] Update demo3d with show flag Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 1a93206..609a21a 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -33,6 +33,12 @@ int main(int argc, char *argv[]) { int n_classes = 3; if(argc > 4) n_classes = atoi(argv[4]); + bool show = false; + if(argc > 5) + show = atoi(argv[5]); + + if(!show) + SAVE_RESULT = true; tk::dnn::CenternetDetection3D cnet; tk::dnn::CenternetDetection3DTrack ctrack; @@ -70,7 +76,8 @@ int main(int argc, char *argv[]) { cv::Mat frame; cv::Mat dnn_input; - cv::namedWindow("detection", cv::WINDOW_NORMAL); + if(show) + cv::namedWindow("detection", cv::WINDOW_NORMAL); std::vector detected_bbox; @@ -86,9 +93,11 @@ int main(int argc, char *argv[]) { //inference detNN->update(dnn_input); frame = detNN->draw(frame); - - cv::imshow("detection", frame); - cv::waitKey(1); + + if(show) { + cv::imshow("detection", frame); + cv::waitKey(1); + } if(SAVE_RESULT) resultVideo << frame; } From 1cfa199ee6b301fafd707b7e0d600423b73357d0 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 9 Dec 2020 19:43:51 +0100 Subject: [PATCH 12/33] Add pre-processing and post-processing stats Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 13 +++++++++++++ include/tkDNN/DetectionNN3D.h | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 609a21a..4286faa 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -105,11 +105,24 @@ int main(int argc, char *argv[]) { std::cout<<"detection end\n"; double mean = 0; + std::cout<pre_stats.begin(), detNN->pre_stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(detNN->pre_stats.begin(), detNN->pre_stats.end())<<" ms\n"; + for(int i=0; ipre_stats.size(); i++) mean += detNN->pre_stats[i]; mean /= detNN->pre_stats.size(); + std::cout<<"Avg: "<stats.begin(), detNN->stats.end())<<" ms\n"; std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n"; for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); std::cout<<"Avg: "<post_stats.begin(), detNN->post_stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(detNN->post_stats.begin(), detNN->post_stats.end())<<" ms\n"; + for(int i=0; ipost_stats.size(); i++) mean += detNN->post_stats[i]; mean /= detNN->post_stats.size(); + std::cout<<"Avg: "< detected; /*bounding boxes in output*/ - std::vector stats; /*keeps track of inference times (ms)*/ + std::vector pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/ std::vector classesNames; DetectionNN3D() {}; @@ -107,6 +107,7 @@ class DetectionNN3D { TKDNN_TSTART preprocess(frame); TKDNN_TSTOP + pre_stats.push_back(t_ns); if(save_times) *times< Date: Mon, 14 Dec 2020 15:32:35 +0100 Subject: [PATCH 13/33] Fix a bug in the 3D bounding boxes. Signed-off-by: Davide Sapienza --- src/CenternetDetection3DTrack.cpp | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 6b34571..ef3e161 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -267,8 +267,8 @@ bool CenternetDetection3DTrack::init_visualization(const int n_classes){ face_id.push_back({0,1,5,4}); face_id.push_back({1,2,6, 5}); - face_id.push_back({2,3,7,6}); face_id.push_back({3,0,4,7}); + face_id.push_back({2,3,7,6}); // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } @@ -809,20 +809,20 @@ cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) { aus.release(); for(int ind_f = 3; ind_f>=0; ind_f--) { for(int j=0; j<4; j++) { - cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(j) * 2), - res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point(res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), + (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), tr_colors[tr_res[i].color], 2); - if(ind_f == 0) { - cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(0) * 2), - res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point(res_corners.at(face_id.at(ind_f).at(2) * 2), - res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2); - cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(1) * 2), - res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point(res_corners.at(face_id.at(ind_f).at(3) * 2), - res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2); + if(ind_f == 0 && j==3) { + cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), + (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), + (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2); + cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), + (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), + (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2); } } } @@ -838,8 +838,9 @@ cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) { bb3=res_corners[2*k+1]; } - cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), - tr_colors[tr_res[i].color], thickness); + // if(not no_bbox): + // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), + // tr_colors[tr_res[i].color], thickness); cv::rectangle(frame, cv::Point(bb0, bb2 - text_size.height - thickness), cv::Point(bb0 + text_size.width, bb2), tr_colors[tr_res[i].color], -1); From be6ad27c11f85481576037658fda7e97005340c9 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Thu, 29 Apr 2021 11:13:24 +0200 Subject: [PATCH 14/33] Batch size > 1 for the 3D demo. This commit lets to use differtent batch size for 3D CenterNet and CenterTrack. Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 71 ++++-- include/tkDNN/CenternetDetection3D.h | 12 +- include/tkDNN/CenternetDetection3DTrack.h | 13 +- include/tkDNN/DetectionNN3D.h | 83 +++--- src/CenternetDetection3D.cpp | 101 ++++---- src/CenternetDetection3DTrack.cpp | 291 +++++++++++----------- 6 files changed, 309 insertions(+), 262 deletions(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 4286faa..c35ac51 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -1,7 +1,7 @@ #include #include #include /* srand, rand */ -#include +//#include #include #include "CenternetDetection3D.h" @@ -24,7 +24,12 @@ int main(int argc, char *argv[]) { std::string net = "dla34_cnet3d_fp32.rt"; if(argc > 1) net = argv[1]; - std::string input = "../demo/yolo_test.mp4"; + #ifdef __linux__ + std::string input = "../demo/yolo_test.mp4"; + #elif _WIN32 + std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + #endif + if(argc > 2) input = argv[2]; char ntype = 'c'; @@ -33,9 +38,18 @@ int main(int argc, char *argv[]) { int n_classes = 3; if(argc > 4) n_classes = atoi(argv[4]); - bool show = false; + int n_batch = 1; if(argc > 5) - show = atoi(argv[5]); + n_batch = atoi(argv[5]); + bool show = true; + if(argc > 6) + show = atoi(argv[6]); + float conf_thresh=0.3; + if(argc > 7) + conf_thresh = atof(argv[7]); + + if(n_batch < 1 || n_batch > 64) + FatalError("Batch dim not supported"); if(!show) SAVE_RESULT = true; @@ -57,7 +71,7 @@ int main(int argc, char *argv[]) { FatalError("Network type not allowed (3rd parameter)\n"); } - detNN->init(net, n_classes); + detNN->init(net, n_classes, n_batch, conf_thresh); gRun = true; @@ -75,30 +89,40 @@ int main(int argc, char *argv[]) { } cv::Mat frame; - cv::Mat dnn_input; if(show) - cv::namedWindow("detection", cv::WINDOW_NORMAL); + cv::namedWindow("detection", cv::WINDOW_NORMAL); - std::vector detected_bbox; + std::vector batch_frame; + std::vector batch_dnn_input; while(gRun) { - cap >> frame; - if(!frame.data) { - break; - } - - // this will be resized to the net format - dnn_input = frame.clone(); + batch_dnn_input.clear(); + batch_frame.clear(); + for(int bi=0; bi< n_batch; ++bi){ + cap >> frame; + if(!frame.data) + break; + + batch_frame.push_back(frame); + + // this will be resized to the net format + batch_dnn_input.push_back(frame.clone()); + } + if(!frame.data) + break; + //inference - detNN->update(dnn_input); - frame = detNN->draw(frame); - - if(show) { - cv::imshow("detection", frame); - cv::waitKey(1); - } - if(SAVE_RESULT) + detNN->update(batch_dnn_input, n_batch); + detNN->draw(batch_frame); + + if(show){ + for(int bi=0; bi< n_batch; ++bi){ + cv::imshow("detection", batch_frame[bi]); + cv::waitKey(1); + } + } + if(n_batch == 1 && SAVE_RESULT) resultVideo << frame; } @@ -124,7 +148,6 @@ int main(int argc, char *argv[]) { std::cout<<"Avg: "< detected3D; - std::vectorcls3D; std::vector> face_id; public: CenternetDetection3D() {}; ~CenternetDetection3D() {}; - bool init(const std::string& tensor_path, const int n_classes=3); - void preprocess(cv::Mat &frame); - void postprocess(); - cv::Mat draw(cv::Mat &frame); + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); + void preprocess(cv::Mat &frame, const int bi=0); + void postprocess(const int bi=0,const bool mAP=false); + void draw(std::vector& frames); }; diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 5149d70..809d0c9 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -145,6 +145,7 @@ private: int count_det; //tracks std::vector tr_res; + std::vector> batchTracked; int count_tr; int track_id=0; @@ -153,7 +154,7 @@ private: bool init_pre_inf(); bool init_postprocessing(); bool init_visualization(const int n_classes); - void pre_inf(); + void pre_inf(const int bi); void _get_additional_inputs(); cv::Mat transform_preds_with_trans(float x1, float x2); void tracking(); @@ -161,11 +162,11 @@ private: public: tk::dnn::Network *pre_phase_net = nullptr; CenternetDetection3DTrack() {}; - ~CenternetDetection3DTrack() {}; - bool init(const std::string& tensor_path, const int n_classes=3); - void preprocess(cv::Mat &frame); - void postprocess(); - cv::Mat draw(cv::Mat &frame); + ~CenternetDetection3DTrack() {}; + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); + void preprocess(cv::Mat &frame, const int bi=0); + void postprocess(const int bi=0,const bool mAP=false); + void draw(std::vector& frames); }; diff --git a/include/tkDNN/DetectionNN3D.h b/include/tkDNN/DetectionNN3D.h index 2870aa0..65cd728 100644 --- a/include/tkDNN/DetectionNN3D.h +++ b/include/tkDNN/DetectionNN3D.h @@ -3,8 +3,11 @@ #include #include -#include +#include +#ifdef __linux__ #include +#endif + #include #include "utils.h" @@ -30,10 +33,12 @@ class DetectionNN3D { tk::dnn::NetworkRT *netRT = nullptr; dnnType *input_d; - cv::Size originalSize; + std::vector originalSize; cv::Scalar colors[256]; + int nBatches = 1; + #ifdef OPENCV_CUDACONTRIB cv::cuda::GpuMat bgr[3]; cv::cuda::GpuMat imagePreproc; @@ -47,21 +52,26 @@ class DetectionNN3D { * This method preprocess the image, before feeding it to the NN. * * @param frame original frame to adapt for inference. + * @param bi batch index */ - virtual void preprocess(cv::Mat &frame) = 0; + virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; /** * This method postprocess the output of the NN to obtain the correct * boundig boxes. * + * @param bi batch index + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation */ - virtual void postprocess() = 0; + virtual void postprocess(const int bi=0,const bool mAP=false) = 0; public: int classes = 0; float confThreshold = 0.3; /*threshold on the confidence of the boxes*/ - - std::vector detected; /*bounding boxes in output*/ + + std::vector detected3D; /*bounding boxes in output*/ + std::vector> batchDetected; /*bounding boxes in output*/ std::vector pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/ std::vector classesNames; @@ -69,68 +79,79 @@ class DetectionNN3D { ~DetectionNN3D(){}; /** - * Method used to inialize the class, allocate memory and compute + * Method used to initialize the class, allocate memory and compute * needed data. * - * @param tensor_path path to the rt file og the NN. + * @param tensor_path path to the rt file of the NN. * @param n_classes number of classes for the given dataset. + * @param n_batches maximum number of batches to use in inference. * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path, const int n_classes=3) = 0; - - /** - * Method to draw boundixg boxes and labels on a frame. - * - * @param frame orginal frame to draw bounding box on. - * @return frame with boundig boxes. - */ - virtual cv::Mat draw(cv::Mat &frame){}; + virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3) = 0; /** * This method performs the whole detection of the NN. * - * @param frame frame to run detection on. + * @param frames frames to run detection on. + * @param cur_batches number of batches to use in inference. * @param save_times if set to true, preprocess, inference and postprocess times * are saved on a csv file, otherwise not. - * @param times pointer to the output stream where to write times + * @param times pointer to the output stream where to write times. + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation. */ - void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){ - if(!frame.data) - FatalError("No image data feed to detection"); - + void update(std::vector& frames, const int cur_batches=1, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ if(save_times && times==nullptr) FatalError("save_times set to true, but no valid ofstream given"); + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); - originalSize = frame.size(); - printCenteredTitle(" TENSORRT detection ", '=', 30); + originalSize.clear(); + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); { TKDNN_TSTART - preprocess(frame); + for(int bi=0; biinput_dim; + dim.n = cur_batches; { - dim.print(); + if(TKDNN_VERBOSE) dim.print(); TKDNN_TSTART netRT->infer(dim, input_d); TKDNN_TSTOP - dim.print(); + if(TKDNN_VERBOSE) dim.print(); stats.push_back(t_ns); if(save_times) *times<& frames){}; + }; }} diff --git a/src/CenternetDetection3D.cpp b/src/CenternetDetection3D.cpp index 1803381..53b3cf7 100644 --- a/src/CenternetDetection3D.cpp +++ b/src/CenternetDetection3D.cpp @@ -3,10 +3,12 @@ namespace tk { namespace dnn { -bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes){ +bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; + nBatches = n_batches; + confThreshold = conf_thresh; dim = netRT->input_dim; @@ -28,7 +30,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas trans = cv::Mat(cv::Size(3,2), CV_32F); trans2 = cv::Mat(cv::Size(3,2), CV_32F); - checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); dim_hm = tk::dnn::dataDim_t(1, 3, 128, 128, 1); dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); @@ -91,7 +93,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); #else - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); mean << 0.485, 0.456, 0.406; stddev << 0.229, 0.224, 0.225; #endif @@ -154,13 +156,13 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } -void CenternetDetection3D::preprocess(cv::Mat &frame){ +void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){ // -----------------------------------pre-process ------------------------------------------ // auto start_t = std::chrono::steady_clock::now(); // auto step_t = std::chrono::steady_clock::now(); // auto end_t = std::chrono::steady_clock::now(); - cv::Size sz = originalSize; + cv::Size sz = originalSize[bi]; // std::cout<<"image: "<(end_t - step_t).count() << " us" << std::endl; // step_t = end_t; - checkCuda(cudaMemcpy(input_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); // end_t = std::chrono::steady_clock::now(); // std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; @@ -280,21 +282,21 @@ void CenternetDetection3D::preprocess(cv::Mat &frame){ int idx = i*imageF.rows*imageF.cols; int ch = dim2.c-3 +i; // std::cout<<"i: "<input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType)); } - checkCuda(cudaMemcpyAsync(input_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice)); + checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice)); #endif } -void CenternetDetection3D::postprocess(){ +void CenternetDetection3D::postprocess(const int bi, const bool mAP) { dnnType *rt_out[7]; - rt_out[0] = (dnnType *)netRT->buffersRT[1]; - rt_out[1] = (dnnType *)netRT->buffersRT[2]; - rt_out[2] = (dnnType *)netRT->buffersRT[3]; - rt_out[3] = (dnnType *)netRT->buffersRT[4]; - rt_out[4] = (dnnType *)netRT->buffersRT[5]; - rt_out[5] = (dnnType *)netRT->buffersRT[6]; - rt_out[6] = (dnnType *)netRT->buffersRT[7]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi; + rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi; + rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi; + rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi; + rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi; + rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi; // ------------------------------------ process -------------------------------------------- activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot()); @@ -404,8 +406,7 @@ void CenternetDetection3D::postprocess(){ if(rot_y peakThreshold) { - if(scores[j] > centerThreshold) { + if(scores[j] > confThreshold) { if(z>0) { // compute_box_3d r.at(0,0) = std::cos(rot_y); @@ -457,16 +458,17 @@ void CenternetDetection3D::postprocess(){ } res.cl = i; res.prob = scores[j]; - res.print(); + //res.print(); detected3D.push_back(res); } } } } } + batchDetected.push_back(detected3D); } -cv::Mat CenternetDetection3D::draw(cv::Mat &frame) { +void CenternetDetection3D::draw(std::vector& frames) { tk::dnn::box3D b; int x0, w, x1, y0, h, y1; int objClass; @@ -476,40 +478,41 @@ cv::Mat CenternetDetection3D::draw(cv::Mat &frame) { float font_scale = 0.5; int thickness = 2; - // draw dets - for(int i=0; i=0; ind_f--) { - for(int j=0; j<4; j++) { - cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), - b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), - colors[b.cl], 2); - if(ind_f == 0) { - cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), - b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), - b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); - cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), - b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), - b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), + b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + colors[b.cl], 2); + if(ind_f == 0) { + cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), + b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), + b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); + cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), + b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), + b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + } } } + // draw label + cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); + cv::rectangle(frames[bi], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1)), + cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), + (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); + cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); } - // draw label - cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); - cv::rectangle(frame, cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1)), - cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), - (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); - cv::putText(frame, classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), - cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); } - return frame; } }} diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index ef3e161..dfc38f9 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -3,12 +3,14 @@ namespace tk { namespace dnn { -bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes){ - std::cout<<(tensor_path).c_str()<<"\n"; + +bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); dim = netRT->input_dim; dim.c = 3; + nBatches = n_batches; + confThreshold = conf_thresh; init_preprocessing(); init_pre_inf(); @@ -46,13 +48,13 @@ bool CenternetDetection3DTrack::init_preprocessing(){ checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); #else - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot())); + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot() * nBatches)); mean << 0.40789655, 0.44719303, 0.47026116; stddev << 0.2886383, 0.27408165, 0.27809834; #endif - checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); checkCuda(cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); } @@ -276,20 +278,20 @@ void CenternetDetection3DTrack::_get_additional_inputs(){ //None no additional input } -void CenternetDetection3DTrack::pre_inf(){ +void CenternetDetection3DTrack::pre_inf(const int bi){ TKDNN_TSTART tk::dnn::dataDim_t dim_aus; pre_phase_net->infer(dim_aus, nullptr); TKDNN_TSTOP checkCuda( cudaDeviceSynchronize() ); - checkCuda( cudaMemcpy(input_d, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); checkCuda( cudaDeviceSynchronize() ); } -void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ - // -----------------------------------pre-process ------------------------------------------ - - cv::Size sz = originalSize; +void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ + // -----------------------------------pre-process ------------------------------------------ + batchTracked.clear(); + cv::Size sz = originalSize[bi]; cv::Size sz_old; float scale = 1.0; float new_height = sz.height * scale; @@ -302,7 +304,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ // float s = new_width >= new_height ? new_width : new_height; // ----------- get_affine_transform // rot_rad = pi * 0 / 100 --> 0 - dim.print(); + //dim.print(); src.at(0,0)=c[0]; src.at(0,1)=c[1]; src.at(1,0)=c[0]; @@ -389,7 +391,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ checkCuda( cudaDeviceSynchronize() ); iter0=false; } - pre_inf(); + pre_inf(bi); checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); checkCuda( cudaDeviceSynchronize() ); @@ -587,17 +589,17 @@ void CenternetDetection3DTrack::tracking(){ } -void CenternetDetection3DTrack::postprocess(){ +void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { dnnType *rt_out[9]; - rt_out[0] = (dnnType *)netRT->buffersRT[1]; - rt_out[1] = (dnnType *)netRT->buffersRT[2]; - rt_out[2] = (dnnType *)netRT->buffersRT[3]; - rt_out[3] = (dnnType *)netRT->buffersRT[4]; - rt_out[4] = (dnnType *)netRT->buffersRT[5]; - rt_out[5] = (dnnType *)netRT->buffersRT[6]; - rt_out[6] = (dnnType *)netRT->buffersRT[7]; - rt_out[7] = (dnnType *)netRT->buffersRT[8]; - rt_out[8] = (dnnType *)netRT->buffersRT[9]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi; + rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi; + rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi; + rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi; + rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi; + rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi; + rt_out[7] = (dnnType *)netRT->buffersRT[8]+ netRT->buffersDIM[8].tot()*bi; + rt_out[8] = (dnnType *)netRT->buffersRT[9]+ netRT->buffersDIM[9].tot()*bi; // ------------------------------------ process -------------------------------------------- @@ -719,143 +721,144 @@ void CenternetDetection3DTrack::postprocess(){ } // track step tracking(); + batchTracked.push_back(tr_res); } -cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) { - +void CenternetDetection3DTrack::draw(std::vector& frames) { + struct trackingRes t; float sc; int id; std::string txt; int baseline = 0; float font_scale = 0.8; - int thickness = 2; - for(int i=0; i vis_thresh){// && tr_res[i].active!=0) { - if(view2d) { - - - cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), tr_res[i].det_res.bb0.at(0,1)), - cv::Point(tr_res[i].det_res.bb1.at(0,0), tr_res[i].det_res.bb1.at(0,1)), tr_colors[tr_res[i].color], thickness); - cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), - tr_res[i].det_res.bb0.at(0,1) - text_size.height - thickness), - cv::Point(tr_res[i].det_res.bb0.at(0,0) + text_size.width, - tr_res[i].det_res.bb0.at(0,1)), tr_colors[tr_res[i].color], -1); - - cv::putText(frame, txt, cv::Point(tr_res[i].det_res.bb0.at(0,0), - tr_res[i].det_res.bb0.at(0,1) - thickness -1), - cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); + int thickness = 2; + for(int bi=0; bi vis_thresh){// && t.active!=0) { + if(view2d) { + cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), t.det_res.bb0.at(0,1)), + cv::Point(t.det_res.bb1.at(0,0), t.det_res.bb1.at(0,1)), tr_colors[t.color], thickness); + cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), + t.det_res.bb0.at(0,1) - text_size.height - thickness), + cv::Point(t.det_res.bb0.at(0,0) + text_size.width, + t.det_res.bb0.at(0,1)), tr_colors[t.color], -1); + + cv::putText(frames[bi], txt, cv::Point(t.det_res.bb0.at(0,0), + t.det_res.bb0.at(0,1) - thickness -1), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frame, cv::Point((int)tr_res[i].det_res.ct.at(0,0), - (int)tr_res[i].det_res.ct.at(0,1)), - cv::Point((int)(tr_res[i].det_res.ct.at(0,0) + tr_res[i].det_res.tr.at(0,0)), - (int)(tr_res[i].det_res.ct.at(0,1) + tr_res[i].det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); - } - //3d - if(!view2d && tr_res[i].det_res.z > 1){ - r.at(0,0) = std::cos(tr_res[i].det_res.rot_y); - r.at(0,2) = std::sin(tr_res[i].det_res.rot_y); - r.at(2,0) = -std::sin(tr_res[i].det_res.rot_y); - r.at(2,2) = std::cos(tr_res[i].det_res.rot_y); - - corners.at(0,0) = tr_res[i].det_res.dim[2]/2; - corners.at(0,1) = tr_res[i].det_res.dim[2]/2; - corners.at(0,2) = -tr_res[i].det_res.dim[2]/2; - corners.at(0,3) = -tr_res[i].det_res.dim[2]/2; - corners.at(0,4) = tr_res[i].det_res.dim[2]/2; - corners.at(0,5) = tr_res[i].det_res.dim[2]/2; - corners.at(0,6) = -tr_res[i].det_res.dim[2]/2; - corners.at(0,7) = -tr_res[i].det_res.dim[2]/2; - - corners.at(1,4) = -tr_res[i].det_res.dim[0]; - corners.at(1,5) = -tr_res[i].det_res.dim[0]; - corners.at(1,6) = -tr_res[i].det_res.dim[0]; - corners.at(1,7) = -tr_res[i].det_res.dim[0]; - - corners.at(2,0) = tr_res[i].det_res.dim[1]/2; - corners.at(2,1) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,2) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,3) = tr_res[i].det_res.dim[1]/2; - corners.at(2,4) = tr_res[i].det_res.dim[1]/2; - corners.at(2,5) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,6) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,7) = tr_res[i].det_res.dim[1]/2; - - cv::Mat aus = r * corners; - - for(int k=0; k<8; k++) { - aus.at(0,k) += tr_res[i].det_res.x; - aus.at(1,k) += tr_res[i].det_res.y; - aus.at(2,k) += tr_res[i].det_res.z; + cv::arrowedLine(frames[bi], cv::Point((int)t.det_res.ct.at(0,0), + (int)t.det_res.ct.at(0,1)), + cv::Point((int)(t.det_res.ct.at(0,0) + t.det_res.tr.at(0,0)), + (int)(t.det_res.ct.at(0,1) + t.det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); } - - // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); - for(int k1=0; k1<3; k1++) { - for(int k2=0; k2<8; k2++) - pts3DHomo.at(k1,k2) = aus.at(k1,k2); - } - - aus.release(); - aus = calibs * pts3DHomo; - std::vector res_corners; - for(int k=0; k<8; k++) { - res_corners.push_back(aus.at(0,k) / aus.at(2,k)); - res_corners.push_back(aus.at(1,k) / aus.at(2,k)); - } - aus.release(); - for(int ind_f = 3; ind_f>=0; ind_f--) { - for(int j=0; j<4; j++) { - cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), - (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), - tr_colors[tr_res[i].color], 2); - if(ind_f == 0 && j==3) { - cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), - (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), - (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2); - cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), - (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), - (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2); + //3d + if(!view2d && t.det_res.z > 1){ + r.at(0,0) = std::cos(t.det_res.rot_y); + r.at(0,2) = std::sin(t.det_res.rot_y); + r.at(2,0) = -std::sin(t.det_res.rot_y); + r.at(2,2) = std::cos(t.det_res.rot_y); + + corners.at(0,0) = t.det_res.dim[2]/2; + corners.at(0,1) = t.det_res.dim[2]/2; + corners.at(0,2) = -t.det_res.dim[2]/2; + corners.at(0,3) = -t.det_res.dim[2]/2; + corners.at(0,4) = t.det_res.dim[2]/2; + corners.at(0,5) = t.det_res.dim[2]/2; + corners.at(0,6) = -t.det_res.dim[2]/2; + corners.at(0,7) = -t.det_res.dim[2]/2; + + corners.at(1,4) = -t.det_res.dim[0]; + corners.at(1,5) = -t.det_res.dim[0]; + corners.at(1,6) = -t.det_res.dim[0]; + corners.at(1,7) = -t.det_res.dim[0]; + + corners.at(2,0) = t.det_res.dim[1]/2; + corners.at(2,1) = -t.det_res.dim[1]/2; + corners.at(2,2) = -t.det_res.dim[1]/2; + corners.at(2,3) = t.det_res.dim[1]/2; + corners.at(2,4) = t.det_res.dim[1]/2; + corners.at(2,5) = -t.det_res.dim[1]/2; + corners.at(2,6) = -t.det_res.dim[1]/2; + corners.at(2,7) = t.det_res.dim[1]/2; + + cv::Mat aus = r * corners; + + for(int k=0; k<8; k++) { + aus.at(0,k) += t.det_res.x; + aus.at(1,k) += t.det_res.y; + aus.at(2,k) += t.det_res.z; + } + + // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); + for(int k1=0; k1<3; k1++) { + for(int k2=0; k2<8; k2++) + pts3DHomo.at(k1,k2) = aus.at(k1,k2); + } + + aus.release(); + aus = calibs * pts3DHomo; + std::vector res_corners; + for(int k=0; k<8; k++) { + res_corners.push_back(aus.at(0,k) / aus.at(2,k)); + res_corners.push_back(aus.at(1,k) / aus.at(2,k)); + } + aus.release(); + for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), + (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + tr_colors[t.color], 2); + if(ind_f == 0 && j==3) { + cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), + (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), + (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[t.color], 2); + cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), + (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), + (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[t.color], 2); + } } } - } - float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; - for(int k=0; k<8; k++) { - if(res_corners[2*k]bb1) - bb1=res_corners[2*k]; - if(res_corners[2*k+1]bb3) - bb3=res_corners[2*k+1]; - - } - // if(not no_bbox): - // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), - // tr_colors[tr_res[i].color], thickness); - cv::rectangle(frame, cv::Point(bb0, bb2 - text_size.height - thickness), - cv::Point(bb0 + text_size.width, bb2), tr_colors[tr_res[i].color], -1); - - cv::putText(frame, txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, - font_scale, cv::Scalar(255, 255, 255), 1); + float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; + for(int k=0; k<8; k++) { + if(res_corners[2*k]bb1) + bb1=res_corners[2*k]; + if(res_corners[2*k+1]bb3) + bb3=res_corners[2*k+1]; + + } + // if(not no_bbox): + // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), + // tr_colors[t.color], thickness); + cv::rectangle(frames[bi], cv::Point(bb0, bb2 - text_size.height - thickness), + cv::Point(bb0 + text_size.width, bb2), tr_colors[t.color], -1); + + cv::putText(frames[bi], txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, + font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frame, cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), - cv::Point((int)((bb0 + bb1)/2 + tr_res[i].det_res.tr.at(0,0)), - (int)((bb2 + bb3)/2 + tr_res[i].det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); + cv::arrowedLine(frames[bi], cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), + cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at(0,0)), + (int)((bb2 + bb3)/2 + t.det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); + } } } - } - return frame; } }} From 2367519799ef3eb9806387dab94ec12d17e77649 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 30 Apr 2021 17:10:51 +0200 Subject: [PATCH 15/33] Add the calibration matrix reading for CenterTrack Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 25 +++++++-- include/tkDNN/CenternetDetection3D.h | 4 +- include/tkDNN/CenternetDetection3DTrack.h | 10 ++-- include/tkDNN/DetectionNN3D.h | 10 ++-- src/CenternetDetection3D.cpp | 5 +- src/CenternetDetection3DTrack.cpp | 65 +++++++++++++---------- 6 files changed, 75 insertions(+), 44 deletions(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index c35ac51..558e6af 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -70,8 +70,17 @@ int main(int argc, char *argv[]) { default: FatalError("Network type not allowed (3rd parameter)\n"); } - - detNN->init(net, n_classes, n_batch, conf_thresh); + std::vector calibs; + // cv::Mat calib = cv::Mat::zeros(cv::Size(3,3), CV_32F); + // calib.at(0,0) = 864.1243196486207;// * 512.0;//884.081444212;//864.1243196486207 * 512.0;// 633.0; + // calib.at(0,2) = 726.7271690557819;// * 512.0;//0.0;//726.7271690557819 * 512.0;// 0.0; //w/2 + // calib.at(1,1) = 883.6552349216504;// * 512.0;//884.081444212;//883.6552349216504 * 512.0;// 633.0; + // calib.at(1,2) = 506.8548506986564;// * 512.0;//0.0;//506.8548506986564 * 512.0;// 0.0; //h/2 + // calibs.push_back(calib); + // calibs.push_back(calib); + // calibs.push_back(calib); + // calibs.push_back(calib); + detNN->init(net, n_classes, n_batch, conf_thresh, calibs); gRun = true; @@ -87,7 +96,8 @@ int main(int argc, char *argv[]) { int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); } - + cv::Size sz_resize = cv::Size(512,512); + std::vector sz_orig; cv::Mat frame; if(show) cv::namedWindow("detection", cv::WINDOW_NORMAL); @@ -98,12 +108,15 @@ int main(int argc, char *argv[]) { while(gRun) { batch_dnn_input.clear(); batch_frame.clear(); + sz_orig.clear(); for(int bi=0; bi< n_batch; ++bi){ cap >> frame; if(!frame.data) break; - + sz_orig.push_back(frame.size()); + if(calibs.size() != 0) + resize(frame, frame, sz_resize); batch_frame.push_back(frame); // this will be resized to the net format @@ -113,11 +126,13 @@ int main(int argc, char *argv[]) { break; //inference - detNN->update(batch_dnn_input, n_batch); + detNN->update(batch_dnn_input, n_batch, false, nullptr, false, sz_orig); detNN->draw(batch_frame); if(show){ for(int bi=0; bi< n_batch; ++bi){ + if(calibs.size() != 0) + resize(batch_frame[bi], batch_frame[bi], sz_orig[bi]); cv::imshow("detection", batch_frame[bi]); cv::waitKey(1); } diff --git a/include/tkDNN/CenternetDetection3D.h b/include/tkDNN/CenternetDetection3D.h index 668440c..cbffa22 100644 --- a/include/tkDNN/CenternetDetection3D.h +++ b/include/tkDNN/CenternetDetection3D.h @@ -82,8 +82,8 @@ public: CenternetDetection3D() {}; ~CenternetDetection3D() {}; - bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); - void preprocess(cv::Mat &frame, const int bi=0); + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); + void preprocess(cv::Mat &frame, const int bi=0, const std::vector& stream_size=std::vector()); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); }; diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 809d0c9..451bf6e 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -74,6 +74,10 @@ private: #endif float *d_ptrs; + std::vector inputCalibs; + + std::vector sz_old; + cv::Mat src; cv::Mat dst; cv::Mat dst2; @@ -124,7 +128,7 @@ private: /* visualization */ cv::Mat r; - cv::Mat calibs; + std::vector calibs; cv::Mat corners, pts3DHomo; std::vector> face_id; @@ -163,8 +167,8 @@ public: tk::dnn::Network *pre_phase_net = nullptr; CenternetDetection3DTrack() {}; ~CenternetDetection3DTrack() {}; - bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); - void preprocess(cv::Mat &frame, const int bi=0); + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); + void preprocess(cv::Mat &frame, const int bi=0, const std::vector& stream_size=std::vector()); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); }; diff --git a/include/tkDNN/DetectionNN3D.h b/include/tkDNN/DetectionNN3D.h index 65cd728..7320bef 100644 --- a/include/tkDNN/DetectionNN3D.h +++ b/include/tkDNN/DetectionNN3D.h @@ -54,7 +54,7 @@ class DetectionNN3D { * @param frame original frame to adapt for inference. * @param bi batch index */ - virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; + virtual void preprocess(cv::Mat &frame, const int bi=0 , const std::vector& stream_size=std::vector()) = 0; /** * This method postprocess the output of the NN to obtain the correct @@ -87,7 +87,8 @@ class DetectionNN3D { * @param n_batches maximum number of batches to use in inference. * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3) = 0; + virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, + const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()) = 0; /** * This method performs the whole detection of the NN. @@ -100,7 +101,8 @@ class DetectionNN3D { * @param mAP set to true only if all the probabilities for a bounding * box are needed, as in some cases for the mAP calculation. */ - void update(std::vector& frames, const int cur_batches=1, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ + void update(std::vector& frames, const int cur_batches=1, bool save_times=false, + std::ofstream *times=nullptr, const bool mAP=false, const std::vector& stream_size=std::vector()){ if(save_times && times==nullptr) FatalError("save_times set to true, but no valid ofstream given"); if(cur_batches > nBatches) @@ -114,7 +116,7 @@ class DetectionNN3D { if(!frames[bi].data) FatalError("No image data feed to detection"); originalSize.push_back(frames[bi].size()); - preprocess(frames[bi], bi); + preprocess(frames[bi], bi, stream_size); } TKDNN_TSTOP pre_stats.push_back(t_ns); diff --git a/src/CenternetDetection3D.cpp b/src/CenternetDetection3D.cpp index 53b3cf7..73e4215 100644 --- a/src/CenternetDetection3D.cpp +++ b/src/CenternetDetection3D.cpp @@ -3,7 +3,8 @@ namespace tk { namespace dnn { -bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { +bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches, + const float conf_thresh, const std::vector& k_calibs) { std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; @@ -156,7 +157,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } -void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){ +void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi, const std::vector& stream_size){ // -----------------------------------pre-process ------------------------------------------ // auto start_t = std::chrono::steady_clock::now(); diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index dfc38f9..02db674 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -4,14 +4,15 @@ namespace tk { namespace dnn { -bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { +bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, + const float conf_thresh, const std::vector& k_calibs) { netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); dim = netRT->input_dim; dim.c = 3; nBatches = n_batches; confThreshold = conf_thresh; - + inputCalibs = k_calibs; init_preprocessing(); init_pre_inf(); init_postprocessing(); @@ -37,7 +38,10 @@ bool CenternetDetection3DTrack::init_preprocessing(){ dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); - + for(int bi=0; bi(0,0) = 633.0; - calibs.at(0,1) = 0.0; - calibs.at(0,2) = 0.0; //w/2 - calibs.at(0,3) = 0.0; - calibs.at(1,0) = 0.0; - calibs.at(1,1) = 633.0; - calibs.at(1,2) = 0.0; //h/2 - calibs.at(1,3) = 0.0; - calibs.at(2,0) = 0.0; - calibs.at(2,1) = 0.0; - calibs.at(2,2) = 1.0; - calibs.at(2,3) = 0.0; + for(int bi=0; bi(0,0) = 633.0; + calibs_.at(1,1) = 633.0; + calibs_.at(2,2) = 1.0; + } + calibs_.at(2,2) = 1.0; + calibs.push_back(calibs_); + } // Alloc array used in the kernel checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); @@ -288,17 +289,25 @@ void CenternetDetection3DTrack::pre_inf(const int bi){ checkCuda( cudaDeviceSynchronize() ); } -void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ +void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const std::vector& stream_size){ // -----------------------------------pre-process ------------------------------------------ batchTracked.clear(); cv::Size sz = originalSize[bi]; - cv::Size sz_old; float scale = 1.0; float new_height = sz.height * scale; float new_width = sz.width * scale; - if(sz.height != sz_old.height && sz.width != sz_old.width){ - calibs.at(0,2) = new_width / 2.0f; - calibs.at(1,2) = new_height /2.0f; + if(sz.height != sz_old[bi].height && sz.width != sz_old[bi].width){ + if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) { + calibs[bi].at(0,2) = new_width / 2.0f; + calibs[bi].at(1,2) = new_height /2.0f; + } + else { + calibs[bi].at(0,0) = inputCalibs[bi].at(0,0) * dim.w / stream_size[bi].width; + calibs[bi].at(0,2) = inputCalibs[bi].at(0,2) * dim.w / stream_size[bi].width; + calibs[bi].at(1,1) = inputCalibs[bi].at(1,1) * dim.h / stream_size[bi].height; + calibs[bi].at(1,2) = inputCalibs[bi].at(1,2) * dim.h / stream_size[bi].height; + } + float c[] = {new_width / 2.0f, new_height /2.0f}; float s[] = {dim.w, dim.h}; // float s = new_width >= new_height ? new_width : new_height; @@ -324,7 +333,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ trans2 = cv::getAffineTransform( dst2, src ); trans2.convertTo(trans_out, CV_32F); } - sz_old = sz; + sz_old[bi] = sz; #ifdef OPENCV_CUDACONTRIB std::cout<<"OPENCV CPMTROB\n"; cv::cuda::GpuMat im_Orig; @@ -358,7 +367,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ #else std::cout<<"NO OPENCV CPMTROB\n"; cv::Mat imageF; - // resize(frame, imageF, cv::Size(new_width, new_height)); + //resize(frame, imageF, cv::Size(512, 512)); imageF = frame; sz = imageF.size(); @@ -701,9 +710,9 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { new_det_res.dim[2] = dim_[i+2*K]; // unproject_2d_to_3d - new_det_res.z = dep[i] - calibs.at(2,3); - new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs.at(0,3) - calibs.at(0,2) * new_det_res.z) / calibs.at(0,0); - new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs.at(1,3) - calibs.at(1,2) * new_det_res.z) / calibs.at(1,1) + (dim_[i] / 2); + new_det_res.z = dep[i] - calibs[bi].at(2,3); + new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs[bi].at(0,3) - calibs[bi].at(0,2) * new_det_res.z) / calibs[bi].at(0,0); + new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs[bi].at(1,3) - calibs[bi].at(1,2) * new_det_res.z) / calibs[bi].at(1,1) + (dim_[i] / 2); // alpha2rot_y // idx = rot[:, 1] > rot[:, 5] @@ -714,7 +723,7 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { new_det_res.alpha = std::atan2(rot[2*K + i], rot[3*K + i]) -0.5 * M_PI; else new_det_res.alpha = std::atan2(rot[6*K + i], rot[7*K + i]) +0.5 * M_PI; - new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at(0,0) - calibs.at(0,2), calibs.at(0,0))); + new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at(0,0) - calibs[bi].at(0,2), calibs[bi].at(0,0))); new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest det_res.push_back(new_det_res); @@ -804,7 +813,7 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { } aus.release(); - aus = calibs * pts3DHomo; + aus = calibs[bi] * pts3DHomo; std::vector res_corners; for(int k=0; k<8; k++) { res_corners.push_back(aus.at(0,k) / aus.at(2,k)); From ff6e0e010adc120bd010c605f82355111758dd08 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 30 Apr 2021 22:26:33 +0200 Subject: [PATCH 16/33] Fix a bug with batch > 1 Signed-off-by: Davide Sapienza --- include/tkDNN/CenternetDetection3DTrack.h | 9 +- src/CenternetDetection3DTrack.cpp | 184 +++++++++++----------- 2 files changed, 99 insertions(+), 94 deletions(-) diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 451bf6e..258fe9d 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -148,10 +148,9 @@ private: std::vector det_res; int count_det; //tracks - std::vector tr_res; - std::vector> batchTracked; - int count_tr; - int track_id=0; + std::vector> tr_res; + std::vector count_tr; + std::vector track_id; bool init_preprocessing(); @@ -161,7 +160,7 @@ private: void pre_inf(const int bi); void _get_additional_inputs(); cv::Mat transform_preds_with_trans(float x1, float x2); - void tracking(); + void tracking(const int bi); public: tk::dnn::Network *pre_phase_net = nullptr; diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 02db674..bdc1c59 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -17,8 +17,6 @@ bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n init_pre_inf(); init_postprocessing(); init_visualization(n_classes); - - count_tr = 0; } bool CenternetDetection3DTrack::init_preprocessing(){ @@ -201,6 +199,11 @@ bool CenternetDetection3DTrack::init_postprocessing(){ // Alloc array used in the kernel checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); + + for(int bi=0; bi& stream_size){ // -----------------------------------pre-process ------------------------------------------ - batchTracked.clear(); cv::Size sz = originalSize[bi]; float scale = 1.0; float new_height = sz.height * scale; @@ -414,8 +416,7 @@ cv::Mat CenternetDetection3DTrack::transform_preds_with_trans(float x1, float x2 return trans_out * target_coords; } -void CenternetDetection3DTrack::tracking(){ - +void CenternetDetection3DTrack::tracking(const int bi) { float item_size[count_det]; int item_cl[count_det]; float dets[2*count_det]; @@ -427,44 +428,44 @@ void CenternetDetection3DTrack::tracking(){ dets[i*2+1] = det_res[i].ct.at(0,1); } - float track_size[count_tr]; - int track_cl[count_tr]; - float tracks[2*count_tr]; - for(int i=0; i(0,0) - tr_res[i].det_res.bb0.at(0,0)) * - (tr_res[i].det_res.bb1.at(0,1) - tr_res[i].det_res.bb0.at(0,1)); - track_cl[i] = tr_res[i].det_res.cl; - tracks[i*2] = tr_res[i].det_res.ct.at(0,0); - tracks[i*2+1] = tr_res[i].det_res.ct.at(0,1); + float track_size[count_tr[bi]]; + int track_cl[count_tr[bi]]; + float tracks[2*count_tr[bi]]; + for(int i=0; i(0,0) - tr_res[bi][i].det_res.bb0.at(0,0)) * + (tr_res[bi][i].det_res.bb1.at(0,1) - tr_res[bi][i].det_res.bb0.at(0,1)); + track_cl[i] = tr_res[bi][i].det_res.cl; + tracks[i*2] = tr_res[bi][i].det_res.ct.at(0,0); + tracks[i*2+1] = tr_res[bi][i].det_res.ct.at(0,1); } - float dist[count_tr*count_det]; + float dist[count_tr[bi]*count_det]; bool invalid; - for(int i=0; i track_size[i] || dist[j*count_tr+i] > item_size[j] || item_cl[j] != track_cl[i]; - dist[j*count_tr+i] = dist[j*count_tr+i] + invalid * (1 << 18); + invalid = dist[j*count_tr[bi]+i] > track_size[i] || dist[j*count_tr[bi]+i] > item_size[j] || item_cl[j] != track_cl[i]; + dist[j*count_tr[bi]+i] = dist[j*count_tr[bi]+i] + invalid * (1 << 18); } } - int matched_indices[2*count_tr]; + int matched_indices[2*count_tr[bi]]; float min_tr; int min_idtr=-1; - for(int i=0; i new_tr_res; int id_new_tr=0; - for(int i=0; i new_thresh) { count_tr_ ++; @@ -583,17 +583,24 @@ void CenternetDetection3DTrack::tracking(){ new_tr_res_.det_res.y = det_res[i].y; new_tr_res_.det_res.z = det_res[i].z; new_tr_res_.det_res.rot_y = det_res[i].rot_y; - new_tr_res_.tracking_id = track_id++; + new_tr_res_.tracking_id = track_id[bi]++; new_tr_res_.age = 1; new_tr_res_.active = 1; new_tr_res_.color = rand() % 256; - tr_res.push_back(new_tr_res_); + if(tr_res.size() <= bi) { + std::vector v_new_tr_res_; + v_new_tr_res_.push_back(new_tr_res_); + tr_res.push_back(v_new_tr_res_); + } + else + tr_res[bi].push_back(new_tr_res_); } } - count_tr = count_tr_; + + count_tr[bi] = count_tr_; - if(track_id==1000) - track_id=0; + if(track_id[bi]==1000) + track_id[bi]=0; det_res.clear(); } @@ -729,8 +736,7 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { } // track step - tracking(); - batchTracked.push_back(tr_res); + tracking(bi); } void CenternetDetection3DTrack::draw(std::vector& frames) { @@ -743,8 +749,8 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { int thickness = 2; for(int bi=0; bi Date: Mon, 3 May 2021 18:56:07 +0200 Subject: [PATCH 17/33] Fix a bug in the draw function of CenterTrack. Signed-off-by: Davide Sapienza --- src/CenternetDetection3DTrack.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index bdc1c59..312bcdb 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -749,7 +749,7 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { int thickness = 2; for(int bi=0; bi Date: Tue, 4 May 2021 11:21:05 +0200 Subject: [PATCH 18/33] Fix tracker for batch size > 1 --- include/tkDNN/CenternetDetection3DTrack.h | 12 +- src/CenternetDetection3DTrack.cpp | 168 +++++++++++----------- 2 files changed, 92 insertions(+), 88 deletions(-) diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 451bf6e..dd092fa 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -1,11 +1,11 @@ #ifndef CENTERNETDETECTION3DTRACK_H #define CENTERNETDETECTION3DTRACK_H +#include +#include "opencv2/opencv.hpp" #include "kernels.h" #include "utils.h" #include "tkdnn.h" -#include -#include "opencv2/opencv.hpp" #include #include #include // std::iota @@ -51,7 +51,7 @@ struct trackingRes class CenternetDetection3DTrack : public DetectionNN3D { -private: +public: tk::dnn::dataDim_t dim; tk::dnn::dataDim_t dim2; tk::dnn::dataDim_t dim_hm; @@ -148,9 +148,9 @@ private: std::vector det_res; int count_det; //tracks - std::vector tr_res; + std::vector> tr_res; std::vector> batchTracked; - int count_tr; + std::vector count_tr; int track_id=0; @@ -161,7 +161,7 @@ private: void pre_inf(const int bi); void _get_additional_inputs(); cv::Mat transform_preds_with_trans(float x1, float x2); - void tracking(); + void tracking(int bi); public: tk::dnn::Network *pre_phase_net = nullptr; diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 02db674..25c4c0b 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -13,12 +13,13 @@ bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n nBatches = n_batches; confThreshold = conf_thresh; inputCalibs = k_calibs; + tr_res.resize(nBatches); init_preprocessing(); init_pre_inf(); init_postprocessing(); init_visualization(n_classes); - count_tr = 0; + count_tr.resize(nBatches, 0); } bool CenternetDetection3DTrack::init_preprocessing(){ @@ -30,6 +31,7 @@ bool CenternetDetection3DTrack::init_preprocessing(){ trans2 = cv::Mat(cv::Size(3,2), CV_32F); trans_out = cv::Mat(cv::Size(3,2), CV_32F); + dst2.at(0,0)=width * 0.5; dst2.at(0,1)=width * 0.5; dst2.at(1,0)=width * 0.5; @@ -372,6 +374,8 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const s sz = imageF.size(); cv::warpAffine(imageF, imageF, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR ); + + cv::imshow("warp", imageF); sz = imageF.size(); imageF.convertTo(imageF, CV_32FC3, 1/255.0); @@ -414,7 +418,7 @@ cv::Mat CenternetDetection3DTrack::transform_preds_with_trans(float x1, float x2 return trans_out * target_coords; } -void CenternetDetection3DTrack::tracking(){ +void CenternetDetection3DTrack::tracking(int bi){ float item_size[count_det]; int item_cl[count_det]; @@ -427,44 +431,44 @@ void CenternetDetection3DTrack::tracking(){ dets[i*2+1] = det_res[i].ct.at(0,1); } - float track_size[count_tr]; - int track_cl[count_tr]; - float tracks[2*count_tr]; - for(int i=0; i(0,0) - tr_res[i].det_res.bb0.at(0,0)) * - (tr_res[i].det_res.bb1.at(0,1) - tr_res[i].det_res.bb0.at(0,1)); - track_cl[i] = tr_res[i].det_res.cl; - tracks[i*2] = tr_res[i].det_res.ct.at(0,0); - tracks[i*2+1] = tr_res[i].det_res.ct.at(0,1); + float track_size[count_tr[bi]]; + int track_cl[count_tr[bi]]; + float tracks[2*count_tr[bi]]; + for(int i=0; i(0,0) - tr_res[bi][i].det_res.bb0.at(0,0)) * + (tr_res[bi][i].det_res.bb1.at(0,1) - tr_res[bi][i].det_res.bb0.at(0,1)); + track_cl[i] = tr_res[bi][i].det_res.cl; + tracks[i*2] = tr_res[bi][i].det_res.ct.at(0,0); + tracks[i*2+1] = tr_res[bi][i].det_res.ct.at(0,1); } - float dist[count_tr*count_det]; + float dist[count_tr[bi]*count_det]; bool invalid; - for(int i=0; i track_size[i] || dist[j*count_tr+i] > item_size[j] || item_cl[j] != track_cl[i]; - dist[j*count_tr+i] = dist[j*count_tr+i] + invalid * (1 << 18); + invalid = dist[j*count_tr[bi]+i] > track_size[i] || dist[j*count_tr[bi]+i] > item_size[j] || item_cl[j] != track_cl[i]; + dist[j*count_tr[bi]+i] = dist[j*count_tr[bi]+i] + invalid * (1 << 18); } } - int matched_indices[2*count_tr]; + int matched_indices[2*count_tr[bi]]; float min_tr; int min_idtr=-1; - for(int i=0; i new_tr_res; int id_new_tr=0; - for(int i=0; i new_thresh) { count_tr_ ++; @@ -587,10 +591,10 @@ void CenternetDetection3DTrack::tracking(){ new_tr_res_.age = 1; new_tr_res_.active = 1; new_tr_res_.color = rand() % 256; - tr_res.push_back(new_tr_res_); + tr_res[bi].push_back(new_tr_res_); } } - count_tr = count_tr_; + count_tr[bi] = count_tr_; if(track_id==1000) track_id=0; @@ -729,8 +733,8 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { } // track step - tracking(); - batchTracked.push_back(tr_res); + tracking(bi); + batchTracked.push_back(tr_res[bi]); } void CenternetDetection3DTrack::draw(std::vector& frames) { From 0dc96d2a9e0070f92d12588f9b9da4b931188a11 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 4 May 2021 17:59:20 +0200 Subject: [PATCH 19/33] Improve CenterTrack. Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 10 +- include/tkDNN/CenternetDetection3D.h | 13 +- include/tkDNN/CenternetDetection3DTrack.h | 30 +- include/tkDNN/DetectionNN3D.h | 6 +- src/CenternetDetection3D.cpp | 107 ++-- src/CenternetDetection3DTrack.cpp | 627 +++++++++++----------- 6 files changed, 397 insertions(+), 396 deletions(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 558e6af..620b0d4 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -96,8 +96,6 @@ int main(int argc, char *argv[]) { int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); } - cv::Size sz_resize = cv::Size(512,512); - std::vector sz_orig; cv::Mat frame; if(show) cv::namedWindow("detection", cv::WINDOW_NORMAL); @@ -108,15 +106,11 @@ int main(int argc, char *argv[]) { while(gRun) { batch_dnn_input.clear(); batch_frame.clear(); - sz_orig.clear(); for(int bi=0; bi< n_batch; ++bi){ cap >> frame; if(!frame.data) break; - sz_orig.push_back(frame.size()); - if(calibs.size() != 0) - resize(frame, frame, sz_resize); batch_frame.push_back(frame); // this will be resized to the net format @@ -126,13 +120,11 @@ int main(int argc, char *argv[]) { break; //inference - detNN->update(batch_dnn_input, n_batch, false, nullptr, false, sz_orig); + detNN->update(batch_dnn_input, n_batch, false, nullptr, false); detNN->draw(batch_frame); if(show){ for(int bi=0; bi< n_batch; ++bi){ - if(calibs.size() != 0) - resize(batch_frame[bi], batch_frame[bi], sz_orig[bi]); cv::imshow("detection", batch_frame[bi]); cv::waitKey(1); } diff --git a/include/tkDNN/CenternetDetection3D.h b/include/tkDNN/CenternetDetection3D.h index cbffa22..943fbf4 100644 --- a/include/tkDNN/CenternetDetection3D.h +++ b/include/tkDNN/CenternetDetection3D.h @@ -27,6 +27,8 @@ private: tk::dnn::dataDim_t dim_dep; tk::dnn::dataDim_t dim_rot; tk::dnn::dataDim_t dim_dim; + + std::vector inputCalibs; float *topk_scores; int *topk_inds_; float *topk_ys_; @@ -58,32 +60,33 @@ private: dnnType *input; #endif cv::Mat r; - cv::Mat calibs; float *d_ptrs; cv::Mat src; cv::Mat dst; cv::Mat dst2; cv::Mat trans, trans2; + std::vector calibs; + //processing int K = 100; int width = 128;//56; // TODO // pointer used in the kernels - float *src_out; - int *ids_out; + float *srcOut; + int *idsOut; struct threshold op; cv::Mat corners, pts3DHomo; - std::vector> face_id; + std::vector> faceId; public: CenternetDetection3D() {}; ~CenternetDetection3D() {}; bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); - void preprocess(cv::Mat &frame, const int bi=0, const std::vector& stream_size=std::vector()); + void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); }; diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 4c036f4..c500412 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -76,12 +76,12 @@ public: std::vector inputCalibs; - std::vector sz_old; + std::vector szOld; cv::Mat src; cv::Mat dst; cv::Mat dst2; - cv::Mat trans, trans2, trans_out; + cv::Mat trans, trans2, transOut; /* pre inf */ bool iter0; @@ -131,27 +131,25 @@ public: std::vector calibs; cv::Mat corners, pts3DHomo; - std::vector> face_id; - cv::Scalar tr_colors[256]; + std::vector> faceId; + cv::Scalar trColors[256]; bool view2d = false; //processing struct threshold op; - float out_thresh = 0.1; - float new_thresh = 0.3; - float vis_thresh = 0.3; - float peakThreshold = 0.2; - float centerThreshold = 0.3; //default 0.5 + float outThresh = 0.1; + float newThresh = 0.3; + // float peakThreshold = 0.2; + // float centerThreshold = 0.3; //default 0.5 //detections - std::vector det_res; - int count_det; + std::vector detRes; + int countDet; //tracks - std::vector> tr_res; - std::vector> batchTracked; - std::vector count_tr; - std::vector track_id; + std::vector> trRes; + std::vector countTr; + std::vector trackId; bool init_preprocessing(); @@ -168,7 +166,7 @@ public: CenternetDetection3DTrack() {}; ~CenternetDetection3DTrack() {}; bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); - void preprocess(cv::Mat &frame, const int bi=0, const std::vector& stream_size=std::vector()); + void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); }; diff --git a/include/tkDNN/DetectionNN3D.h b/include/tkDNN/DetectionNN3D.h index 7320bef..af6eaf9 100644 --- a/include/tkDNN/DetectionNN3D.h +++ b/include/tkDNN/DetectionNN3D.h @@ -54,7 +54,7 @@ class DetectionNN3D { * @param frame original frame to adapt for inference. * @param bi batch index */ - virtual void preprocess(cv::Mat &frame, const int bi=0 , const std::vector& stream_size=std::vector()) = 0; + virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; /** * This method postprocess the output of the NN to obtain the correct @@ -102,7 +102,7 @@ class DetectionNN3D { * box are needed, as in some cases for the mAP calculation. */ void update(std::vector& frames, const int cur_batches=1, bool save_times=false, - std::ofstream *times=nullptr, const bool mAP=false, const std::vector& stream_size=std::vector()){ + std::ofstream *times=nullptr, const bool mAP=false){ if(save_times && times==nullptr) FatalError("save_times set to true, but no valid ofstream given"); if(cur_batches > nBatches) @@ -116,7 +116,7 @@ class DetectionNN3D { if(!frames[bi].data) FatalError("No image data feed to detection"); originalSize.push_back(frames[bi].size()); - preprocess(frames[bi], bi, stream_size); + preprocess(frames[bi], bi); } TKDNN_TSTOP pre_stats.push_back(t_ns); diff --git a/src/CenternetDetection3D.cpp b/src/CenternetDetection3D.cpp index 73e4215..8f7d7c3 100644 --- a/src/CenternetDetection3D.cpp +++ b/src/CenternetDetection3D.cpp @@ -10,7 +10,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas classes = n_classes; nBatches = n_batches; confThreshold = conf_thresh; - + inputCalibs = k_calibs; dim = netRT->input_dim; const char *kitti_class_name[] = { @@ -99,19 +99,26 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas stddev << 0.229, 0.224, 0.225; #endif - calibs = cv::Mat(cv::Size(4,3), CV_32F); - calibs.at(0,0) = 707.0493; - calibs.at(0,1) = 0.0; - calibs.at(0,2) = 604.0814; - calibs.at(0,3) = 45.75831; - calibs.at(1,0) = 0.0; - calibs.at(1,1) = 707.0493; - calibs.at(1,2) = 180.5066; - calibs.at(1,3) = -0.3454157; - calibs.at(2,0) = 0.0; - calibs.at(2,1) = 0.0; - calibs.at(2,2) = 1.0; - calibs.at(2,3) = 0.004981016; + for(int bi=0; bi(0,0) = 707.0493; + calibs_.at(0,2) = 604.0814; + calibs_.at(1,1) = 707.0493; + calibs_.at(1,2) = 180.5066; + } + else { + calibs_.at(0,0) = inputCalibs[bi].at(0,0) * dim.w / 1440; + calibs_.at(0,2) = inputCalibs[bi].at(0,2) * dim.w / 1440; + calibs_.at(1,1) = inputCalibs[bi].at(1,1) * dim.h / 1080; + calibs_.at(1,2) = inputCalibs[bi].at(1,2) * dim.h / 1080; + } + calibs_.at(0,3) = 45.75831; + calibs_.at(1,3) = -0.3454157; + calibs_.at(2,2) = 1.0; + calibs_.at(2,3) = 0.004981016; + calibs.push_back(calibs_); + } r = cv::Mat(cv::Size(3,3), CV_32F); r.at(0,1) = 0.0; @@ -139,8 +146,8 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) ); // Alloc array used in the kernel - checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); - checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); + checkCuda( cudaMalloc(&srcOut, K *sizeof(float)) ); + checkCuda( cudaMalloc(&idsOut, K *sizeof(int)) ); dst2.at(0,0)=width * 0.5; dst2.at(0,1)=width * 0.5; @@ -150,16 +157,14 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); - face_id.push_back({0,1,5,4}); - face_id.push_back({1,2,6, 5}); - face_id.push_back({2,3,7,6}); - face_id.push_back({3,0,4,7}); + faceId.push_back({0,1,5,4}); + faceId.push_back({1,2,6, 5}); + faceId.push_back({2,3,7,6}); + faceId.push_back({3,0,4,7}); // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } -void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi, const std::vector& stream_size){ - // -----------------------------------pre-process ------------------------------------------ - +void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){ // auto start_t = std::chrono::steady_clock::now(); // auto step_t = std::chrono::steady_clock::now(); // auto end_t = std::chrono::steady_clock::now(); @@ -338,20 +343,20 @@ void CenternetDetection3D::postprocess(const int bi, const bool mAP) { // ----------- topk end - topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out); + topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], srcOut, idsOut); // checkCuda( cudaDeviceSynchronize() ); - getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[4], dep_d, ids_out); + getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[4], dep_d, idsOut); checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) ); - getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[5], rot_d, ids_out); + getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[5], rot_d, idsOut); checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) ); - getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[6], dim_d, ids_out); + getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[6], dim_d, idsOut); checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) ); - getRecordsFromTopKId(topk_inds_d, K, dim_wh.c, dim_wh.h * dim_wh.w, rt_out[2], wh_d, ids_out); + getRecordsFromTopKId(topk_inds_d, K, dim_wh.c, dim_wh.h * dim_wh.w, rt_out[2], wh_d, idsOut); checkCuda( cudaMemcpy(wh, wh_d, K * dim_wh.c * sizeof(float), cudaMemcpyDeviceToHost) ); checkCuda( cudaMemcpy(xs, topk_xs_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); @@ -397,11 +402,11 @@ void CenternetDetection3D::postprocess(const int bi, const bool mAP) { alpha = std::atan2(rot[6*K + j], rot[7*K + j]) +0.5 * M_PI; // unproject_2d_to_3d - z = dep[j] - calibs.at(2,3);// z = depth - P[2, 3] - x = (target_coords[j*4] * dep[j] - calibs.at(0,3) - calibs.at(0,2) * z) / calibs.at(0,0); - y = (target_coords[j*4+1] * dep[j] - calibs.at(1,3) - calibs.at(1,2) * z) / calibs.at(1,1) + (dim_[j] / 2); + z = dep[j] - calibs[bi].at(2,3);// z = depth - P[2, 3] + x = (target_coords[j*4] * dep[j] - calibs[bi].at(0,3) - calibs[bi].at(0,2) * z) / calibs[bi].at(0,0); + y = (target_coords[j*4+1] * dep[j] - calibs[bi].at(1,3) - calibs[bi].at(1,2) * z) / calibs[bi].at(1,1) + (dim_[j] / 2); // alpha2rot_y - rot_y = (alpha + std::atan2(target_coords[j*4] - calibs.at(0,2), calibs.at(0,0))); + rot_y = (alpha + std::atan2(target_coords[j*4] - calibs[bi].at(0,2), calibs[bi].at(0,0))); if(rot_y>M_PI) rot_y -= 2*M_PI; if(rot_y(k1,k2) = aus.at(k1,k2); } aus.release(); - aus = calibs * pts3DHomo; + aus = calibs[bi] * pts3DHomo; tk::dnn::box3D res; for(int k=0; k<8; k++) { @@ -486,31 +491,31 @@ void CenternetDetection3D::draw(std::vector& frames) { for(int ind_f = 3; ind_f>=0; ind_f--) { for(int j=0; j<4; j++) { - cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), - b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(j) * 2), + b.corners.at(faceId.at(ind_f).at(j) * 2 + 1)), + cv::Point(b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2), + b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1)), colors[b.cl], 2); if(ind_f == 0) { - cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), - b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), - b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); - cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), - b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), - b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(0) * 2), + b.corners.at(faceId.at(ind_f).at(0) * 2 + 1)), + cv::Point(b.corners.at(faceId.at(ind_f).at(2) * 2), + b.corners.at(faceId.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); + cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(1) * 2), + b.corners.at(faceId.at(ind_f).at(1) * 2 + 1)), + cv::Point(b.corners.at(faceId.at(ind_f).at(3) * 2), + b.corners.at(faceId.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); } } } // draw label cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); - cv::rectangle(frames[bi], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1)), - cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), - (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); - cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), + cv::rectangle(frames[bi], cv::Point(b.corners.at(faceId.at(0).at(0) * 2), + b.corners.at(faceId.at(0).at(0) * 2 + 1)), + cv::Point((b.corners.at(faceId.at(0).at(0) * 2) + text_size.width - 2), + (b.corners.at(faceId.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); + cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(faceId.at(0).at(0) * 2), + b.corners.at(faceId.at(0).at(0) * 2 + 1) - (baseline / 2)), cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); } } diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 3653896..d119c1e 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -6,83 +6,77 @@ namespace tk { namespace dnn { bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh, const std::vector& k_calibs) { - netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); - - dim = netRT->input_dim; - dim.c = 3; - nBatches = n_batches; + netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); + dim = netRT->input_dim; + dim.c = 3; + nBatches = n_batches; confThreshold = conf_thresh; - inputCalibs = k_calibs; - tr_res.resize(nBatches); - count_tr.resize(nBatches, 0); + inputCalibs = k_calibs; init_preprocessing(); init_pre_inf(); init_postprocessing(); init_visualization(n_classes); - } bool CenternetDetection3DTrack::init_preprocessing(){ //image transformation - src = cv::Mat(cv::Size(2,3), CV_32F); - dst = cv::Mat(cv::Size(2,3), CV_32F); - dst2 = cv::Mat(cv::Size(2,3), CV_32F); - trans = cv::Mat(cv::Size(3,2), CV_32F); - trans2 = cv::Mat(cv::Size(3,2), CV_32F); - trans_out = cv::Mat(cv::Size(3,2), CV_32F); + src = cv::Mat(cv::Size(2,3), CV_32F); + dst = cv::Mat(cv::Size(2,3), CV_32F); + dst2 = cv::Mat(cv::Size(2,3), CV_32F); + trans = cv::Mat(cv::Size(3,2), CV_32F); + trans2 = cv::Mat(cv::Size(3,2), CV_32F); + transOut = cv::Mat(cv::Size(3,2), CV_32F); - - dst2.at(0,0)=width * 0.5; - dst2.at(0,1)=width * 0.5; - dst2.at(1,0)=width * 0.5; - dst2.at(1,1)=width * 0.5 + width * -0.5; - - dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); - dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); + dst2.at(0,0) = width * 0.5; + dst2.at(0,1) = width * 0.5; + dst2.at(1,0) = width * 0.5; + dst2.at(1,1) = width * 0.5 + width * -0.5; + dst2.at(2,0) = dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); + dst2.at(2,1) = dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); for(int bi=0; biinput_dim.tot() * nBatches)); - checkCuda(cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); + checkCuda( cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); + checkCuda( cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); } bool CenternetDetection3DTrack::init_pre_inf(){ // initial steps: the first part of the network const char *pre_img_conv1_bin = "dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; - const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; - const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; - const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; + const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; + const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; + const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; dim_in0 = tk::dnn::dataDim_t(1, 3, 512, 512, 1); dim_in1 = tk::dnn::dataDim_t(1, 1, 512, 512, 1); - checkCuda( cudaMalloc(&out_d, netRT->input_dim.tot()*sizeof(dnnType)) ); checkCuda( cudaMalloc(&img_d, dim_in0.tot()*sizeof(dnnType)) ); checkCuda( cudaMalloc(&hm_d, dim_in1.tot()*sizeof(dnnType)) ); // init to zeros hm - dnnType *hm_h; + dnnType *hm_h; checkCuda( cudaMallocHost(&hm_h, 1 * dim.h * dim.w*sizeof(dnnType)) ); for(int i=0; i<1 * dim.h * dim.w; i++) - hm_h[i]=0.0f; + hm_h[i] = 0.0f; checkCuda( cudaMemcpy(hm_d, hm_h, 1 * dim.h * dim.w * sizeof(dnnType), cudaMemcpyHostToDevice) ); checkCuda( cudaFreeHost(hm_h) ); dnnType *i0_h, *i1_h, *i2_h; @@ -97,20 +91,20 @@ bool CenternetDetection3DTrack::init_pre_inf(){ pre_phase_net = new tk::dnn::Network(dim_in0); //pre-img - tk::dnn::Input *in_pre_img = new tk::dnn::Input(pre_phase_net, dim_in0, img_d); - tk::dnn::Conv2d *pre_img_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true); - tk::dnn::Activation *pre_img_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + tk::dnn::Input *in_pre_img = new tk::dnn::Input(pre_phase_net, dim_in0, img_d); + tk::dnn::Conv2d *pre_img_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true); + tk::dnn::Activation *pre_img_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); //pre-hm - tk::dnn::Input *in_pre_hm = new tk::dnn::Input(pre_phase_net, dim_in1, hm_d); - tk::dnn::Conv2d *pre_hm_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true); - tk::dnn::Activation *pre_hm_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + tk::dnn::Input *in_pre_hm = new tk::dnn::Input(pre_phase_net, dim_in1, hm_d); + tk::dnn::Conv2d *pre_hm_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true); + tk::dnn::Activation *pre_hm_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); // image input - tk::dnn::Input *input_image = new tk::dnn::Input(pre_phase_net, dim_in0, input_pre_inf_d); - tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); - tk::dnn::Activation *relu1 = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + tk::dnn::Input *input_image = new tk::dnn::Input(pre_phase_net, dim_in0, input_pre_inf_d); + tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation *relu1 = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); - tk::dnn::Shortcut *s0_input = new tk::dnn::Shortcut(pre_phase_net, pre_img_relu); - tk::dnn::Shortcut *s1_input = new tk::dnn::Shortcut(pre_phase_net, pre_hm_relu); + tk::dnn::Shortcut *s0_input = new tk::dnn::Shortcut(pre_phase_net, pre_img_relu); + tk::dnn::Shortcut *s1_input = new tk::dnn::Shortcut(pre_phase_net, pre_hm_relu); // output data out_d = s1_input->dstData; //print network model @@ -123,14 +117,14 @@ bool CenternetDetection3DTrack::init_pre_inf(){ bool CenternetDetection3DTrack::init_postprocessing(){ srand(0); //seed = 0 for random colors - dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1); - dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); - dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); - dim_track = tk::dnn::dataDim_t(1, 2, 128, 128, 1); - dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1); - dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1); - dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1); - dim_amodel_offset = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1); + dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_track = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1); + dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1); + dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1); + dim_amodel_offset = tk::dnn::dataDim_t(1, 2, 128, 128, 1); checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) ); checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) ); @@ -138,7 +132,7 @@ bool CenternetDetection3DTrack::init_postprocessing(){ checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) ); checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); - for(int i =0; i(coco_class_name, std::end( coco_class_name)); for(int c=0; c(3,6) = 1.0; pts3DHomo.at(3,7) = 1.0; - face_id.push_back({0,1,5,4}); - face_id.push_back({1,2,6, 5}); - face_id.push_back({3,0,4,7}); - face_id.push_back({2,3,7,6}); + faceId.push_back({0,1,5,4}); + faceId.push_back({1,2,6, 5}); + faceId.push_back({3,0,4,7}); + faceId.push_back({2,3,7,6}); // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } @@ -296,22 +289,21 @@ void CenternetDetection3DTrack::pre_inf(const int bi){ checkCuda( cudaDeviceSynchronize() ); } -void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const std::vector& stream_size){ - // -----------------------------------pre-process ------------------------------------------ +void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ cv::Size sz = originalSize[bi]; - float scale = 1.0; - float new_height = sz.height * scale; - float new_width = sz.width * scale; - if(sz.height != sz_old[bi].height && sz.width != sz_old[bi].width){ + // float scale = 1.0; + float new_height = dim.h;//sz.height * scale; + float new_width = dim.w;//sz.width * scale; + if(sz.height != szOld[bi].height && sz.width != szOld[bi].width){ if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) { calibs[bi].at(0,2) = new_width / 2.0f; calibs[bi].at(1,2) = new_height /2.0f; } else { - calibs[bi].at(0,0) = inputCalibs[bi].at(0,0) * dim.w / stream_size[bi].width; - calibs[bi].at(0,2) = inputCalibs[bi].at(0,2) * dim.w / stream_size[bi].width; - calibs[bi].at(1,1) = inputCalibs[bi].at(1,1) * dim.h / stream_size[bi].height; - calibs[bi].at(1,2) = inputCalibs[bi].at(1,2) * dim.h / stream_size[bi].height; + calibs[bi].at(0,0) = inputCalibs[bi].at(0,0) * dim.w / sz.width; + calibs[bi].at(0,2) = inputCalibs[bi].at(0,2) * dim.w / sz.width; + calibs[bi].at(1,1) = inputCalibs[bi].at(1,1) * dim.h / sz.height; + calibs[bi].at(1,2) = inputCalibs[bi].at(1,2) * dim.h / sz.height; } float c[] = {new_width / 2.0f, new_height /2.0f}; @@ -320,34 +312,33 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const s // ----------- get_affine_transform // rot_rad = pi * 0 / 100 --> 0 //dim.print(); - src.at(0,0)=c[0]; - src.at(0,1)=c[1]; - src.at(1,0)=c[0]; - src.at(1,1)=c[1] + s[0] * -0.5; - dst.at(0,0)=dim.w * 0.5; - dst.at(0,1)=dim.h * 0.5; - dst.at(1,0)=dim.w * 0.5; - dst.at(1,1)=dim.h * 0.5 + dim.w * -0.5; + src.at(0,0) = c[0]; + src.at(0,1) = c[1]; + src.at(1,0) = c[0]; + src.at(1,1) = c[1] + s[0] * -0.5; + dst.at(0,0) = dim.w * 0.5; + dst.at(0,1) = dim.h * 0.5; + dst.at(1,0) = dim.w * 0.5; + dst.at(1,1) = dim.h * 0.5 + dim.w * -0.5; - src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); - src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); - dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); - dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); + src.at(2,0) = src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); + src.at(2,1) = src.at(1,1) + (src.at(0,0)-src.at(1,0) ); + dst.at(2,0) = dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); + dst.at(2,1) = dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); trans = cv::getAffineTransform( src, dst ); trans2 = cv::getAffineTransform( dst2, src ); - trans2.convertTo(trans_out, CV_32F); + trans2.convertTo(transOut, CV_32F); } - sz_old[bi] = sz; + szOld[bi] = sz; #ifdef OPENCV_CUDACONTRIB - std::cout<<"OPENCV CPMTROB\n"; cv::cuda::GpuMat im_Orig; cv::cuda::GpuMat imageF1_d, imageF2_d; im_Orig = cv::cuda::GpuMat(frame); - // cv::cuda::resize (im_Orig, imageF1_d, cv::Size(new_width, new_height)); - imageF1_d = im_Orig; + cv::cuda::resize (im_Orig, imageF1_d, cv::Size(dim.w, dim.h)); + // imageF1_d = im_Orig; checkCuda( cudaDeviceSynchronize() ); sz = imageF1_d.size(); @@ -367,20 +358,18 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const s normalize(d_ptrs, dim.c, dim.h, dim.w, mean_d, stddev_d); - checkCuda(cudaMemcpy(input_pre_inf_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + checkCuda( cudaMemcpy(input_pre_inf_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); checkCuda( cudaDeviceSynchronize() ); #else - std::cout<<"NO OPENCV CPMTROB\n"; cv::Mat imageF; - //resize(frame, imageF, cv::Size(512, 512)); - imageF = frame; + resize(frame, imageF, cv::Size(dim.w, dim.h)); + // imageF = frame; sz = imageF.size(); - cv::warpAffine(imageF, imageF, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR ); - //cv::imshow("warp", imageF); - + // cv::imshow("warp", imageF); + sz = imageF.size(); imageF.convertTo(imageF, CV_32FC3, 1/255.0); @@ -394,11 +383,11 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const s bgr[i] = bgr[i] / stddev[i]; } for(int i=0; i(0,0) = x1; target_coords.at(0,1) = x2; target_coords.at(0,2) = 1.0; - return trans_out * target_coords; + return transOut * target_coords; } void CenternetDetection3DTrack::tracking(const int bi) { - float item_size[count_det]; - int item_cl[count_det]; - float dets[2*count_det]; - for(int i=0; i(0,0) - det_res[i].bb0.at(0,0)) * - (det_res[i].bb1.at(0,1) - det_res[i].bb0.at(0,1)); - item_cl[i] = det_res[i].cl; - dets[i*2] = det_res[i].ct.at(0,0); - dets[i*2+1] = det_res[i].ct.at(0,1); + float item_size[countDet]; + int item_cl[countDet]; + float dets[2*countDet]; + for(int i=0; i(0,0) - detRes[i].bb0.at(0,0)) * + (detRes[i].bb1.at(0,1) - detRes[i].bb0.at(0,1)); + item_cl[i] = detRes[i].cl; + dets[i*2] = detRes[i].ct.at(0,0); + dets[i*2+1] = detRes[i].ct.at(0,1); } - float track_size[count_tr[bi]]; - int track_cl[count_tr[bi]]; - float tracks[2*count_tr[bi]]; - for(int i=0; i(0,0) - tr_res[bi][i].det_res.bb0.at(0,0)) * - (tr_res[bi][i].det_res.bb1.at(0,1) - tr_res[bi][i].det_res.bb0.at(0,1)); - track_cl[i] = tr_res[bi][i].det_res.cl; - tracks[i*2] = tr_res[bi][i].det_res.ct.at(0,0); - tracks[i*2+1] = tr_res[bi][i].det_res.ct.at(0,1); + float track_size[countTr[bi]]; + int track_cl[countTr[bi]]; + float tracks[2*countTr[bi]]; + for(int i=0; i(0,0) - trRes[bi][i].det_res.bb0.at(0,0)) * + (trRes[bi][i].det_res.bb1.at(0,1) - trRes[bi][i].det_res.bb0.at(0,1)); + track_cl[i] = trRes[bi][i].det_res.cl; + tracks[i*2] = trRes[bi][i].det_res.ct.at(0,0); + tracks[i*2+1] = trRes[bi][i].det_res.ct.at(0,1); } - float dist[count_tr[bi]*count_det]; + float dist[countTr[bi]*countDet]; bool invalid; - for(int i=0; i track_size[i] || dist[j*count_tr[bi]+i] > item_size[j] || item_cl[j] != track_cl[i]; - dist[j*count_tr[bi]+i] = dist[j*count_tr[bi]+i] + invalid * (1 << 18); + for(int i=0; i track_size[i] || + dist[j*countTr[bi]+i] > item_size[j] || + item_cl[j] != track_cl[i]; + dist[j*countTr[bi]+i] = dist[j*countTr[bi]+i] + invalid * (1 << 18); } } - int matched_indices[2*count_tr[bi]]; + int matched_indices[2*countTr[bi]]; float min_tr; - int min_idtr=-1; - for(int i=0; i new_tr_res; int id_new_tr=0; - for(int i=0; i new_thresh) { + int count_tr_ = countTr[bi]; + for(int i=0; i newThresh) { count_tr_ ++; struct trackingRes new_tr_res_; - new_tr_res_.det_res.score = det_res[i].score; - new_tr_res_.det_res.cl = det_res[i].cl; - new_tr_res_.det_res.ct = det_res[i].ct; - new_tr_res_.det_res.tr = det_res[i].tr; - new_tr_res_.det_res.bb0 = det_res[i].bb0; - new_tr_res_.det_res.bb1 = det_res[i].bb1; - new_tr_res_.det_res.dep = det_res[i].dep; - new_tr_res_.det_res.dim[0] = det_res[i].dim[0]; - new_tr_res_.det_res.dim[1] = det_res[i].dim[1]; - new_tr_res_.det_res.dim[2] = det_res[i].dim[2]; - new_tr_res_.det_res.alpha = det_res[i].alpha; - new_tr_res_.det_res.x = det_res[i].x; - new_tr_res_.det_res.y = det_res[i].y; - new_tr_res_.det_res.z = det_res[i].z; - new_tr_res_.det_res.rot_y = det_res[i].rot_y; - new_tr_res_.tracking_id = track_id[bi]++; - new_tr_res_.age = 1; - new_tr_res_.active = 1; - new_tr_res_.color = rand() % 256; - if(tr_res.size() <= bi) { + new_tr_res_.det_res.score = detRes[i].score; + new_tr_res_.det_res.cl = detRes[i].cl; + new_tr_res_.det_res.ct = detRes[i].ct; + new_tr_res_.det_res.tr = detRes[i].tr; + new_tr_res_.det_res.bb0 = detRes[i].bb0; + new_tr_res_.det_res.bb1 = detRes[i].bb1; + new_tr_res_.det_res.dep = detRes[i].dep; + new_tr_res_.det_res.dim[0] = detRes[i].dim[0]; + new_tr_res_.det_res.dim[1] = detRes[i].dim[1]; + new_tr_res_.det_res.dim[2] = detRes[i].dim[2]; + new_tr_res_.det_res.alpha = detRes[i].alpha; + new_tr_res_.det_res.x = detRes[i].x; + new_tr_res_.det_res.y = detRes[i].y; + new_tr_res_.det_res.z = detRes[i].z; + new_tr_res_.det_res.rot_y = detRes[i].rot_y; + new_tr_res_.tracking_id = trackId[bi]++; + new_tr_res_.age = 1; + new_tr_res_.active = 1; + new_tr_res_.color = rand() % 256; + if(trRes.size() <= bi) { std::vector v_new_tr_res_; v_new_tr_res_.push_back(new_tr_res_); - tr_res.push_back(v_new_tr_res_); + trRes.push_back(v_new_tr_res_); } else - tr_res[bi].push_back(new_tr_res_); + trRes[bi].push_back(new_tr_res_); } } - count_tr[bi] = count_tr_; - - if(track_id[bi]==1000) - track_id[bi]=0; - det_res.clear(); + countTr[bi] = count_tr_; + //reset the tracker id + if(trackId[bi] == 1000) + trackId[bi] = 0; + detRes.clear(); } @@ -697,35 +686,36 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { // ---------------------------------- post-process ----------------------------------------- - count_det = 0; - det_res.clear(); - for(int i = 0; i(2,3); - new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs[bi].at(0,3) - calibs[bi].at(0,2) * new_det_res.z) / calibs[bi].at(0,0); - new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs[bi].at(1,3) - calibs[bi].at(1,2) * new_det_res.z) / calibs[bi].at(1,1) + (dim_[i] / 2); + new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs[bi].at(0,3) - + calibs[bi].at(0,2) * new_det_res.z) / calibs[bi].at(0,0); + new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs[bi].at(1,3) - + calibs[bi].at(1,2) * new_det_res.z) / calibs[bi].at(1,1) + (dim_[i] / 2); // alpha2rot_y // idx = rot[:, 1] > rot[:, 5] @@ -737,13 +727,11 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { else new_det_res.alpha = std::atan2(rot[6*K + i], rot[7*K + i]) +0.5 * M_PI; new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at(0,0) - calibs[bi].at(0,2), calibs[bi].at(0,0))); - new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest - det_res.push_back(new_det_res); - + new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest + detRes.push_back(new_det_res); } // track step tracking(bi); - batchTracked.push_back(tr_res[bi]); } void CenternetDetection3DTrack::draw(std::vector& frames) { @@ -754,32 +742,38 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { int baseline = 0; float font_scale = 0.8; int thickness = 2; + for(int bi=0; bi vis_thresh){// && t.active!=0) { + if(t.det_res.score > confThreshold){// && t.active!=0) { if(view2d) { - cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), t.det_res.bb0.at(0,1)), - cv::Point(t.det_res.bb1.at(0,0), t.det_res.bb1.at(0,1)), tr_colors[t.color], thickness); - cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), - t.det_res.bb0.at(0,1) - text_size.height - thickness), - cv::Point(t.det_res.bb0.at(0,0) + text_size.width, - t.det_res.bb0.at(0,1)), tr_colors[t.color], -1); + cv::rectangle(frames[bi], + cv::Point(t.det_res.bb0.at(0,0) * scale_x, t.det_res.bb0.at(0,1) * scale_y), + cv::Point(t.det_res.bb1.at(0,0) * scale_x, t.det_res.bb1.at(0,1) * scale_y), + trColors[t.color], thickness); + cv::rectangle(frames[bi], + cv::Point(t.det_res.bb0.at(0,0) * scale_x, t.det_res.bb0.at(0,1) * scale_y - text_size.height - thickness), + cv::Point(t.det_res.bb0.at(0,0) * scale_x + text_size.width, t.det_res.bb0.at(0,1) * scale_y), + trColors[t.color], -1); - cv::putText(frames[bi], txt, cv::Point(t.det_res.bb0.at(0,0), - t.det_res.bb0.at(0,1) - thickness -1), - cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); + cv::putText(frames[bi], txt, + cv::Point(t.det_res.bb0.at(0,0) * scale_x, t.det_res.bb0.at(0,1) * scale_y - thickness -1), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frames[bi], cv::Point((int)t.det_res.ct.at(0,0), - (int)t.det_res.ct.at(0,1)), - cv::Point((int)(t.det_res.ct.at(0,0) + t.det_res.tr.at(0,0)), - (int)(t.det_res.ct.at(0,1) + t.det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); + cv::arrowedLine(frames[bi], + cv::Point((int)t.det_res.ct.at(0,0) * scale_x, (int)t.det_res.ct.at(0,1) * scale_y), + cv::Point((int)(t.det_res.ct.at(0,0) * scale_x + t.det_res.tr.at(0,0) * scale_x), + (int)(t.det_res.ct.at(0,1) * scale_y + t.det_res.tr.at(0,1) * scale_y)), + cv::Scalar(255, 0, 255), 2); } //3d if(!view2d && t.det_res.z > 1){ @@ -833,50 +827,59 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { res_corners.push_back(aus.at(1,k) / aus.at(2,k)); } aus.release(); - for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int ind_f=3; ind_f>=0; ind_f--) { for(int j=0; j<4; j++) { - cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), - (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), - tr_colors[t.color], 2); + cv::line(frames[bi], + cv::Point((int)res_corners.at(faceId.at(ind_f).at(j) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(j) * 2 + 1) * scale_y), + cv::Point((int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1) * scale_y), + trColors[t.color], 2); if(ind_f == 0 && j==3) { - cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), - (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), - (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[t.color], 2); - cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), - (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), - (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[t.color], 2); + cv::line(frames[bi], + cv::Point((int)res_corners.at(faceId.at(ind_f).at(0) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(0) * 2 + 1) * scale_y), + cv::Point((int)res_corners.at(faceId.at(ind_f).at(2) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(2) * 2 + 1) * scale_y), trColors[t.color], 2); + cv::line(frames[bi], + cv::Point((int)res_corners.at(faceId.at(ind_f).at(1) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(1) * 2 + 1) * scale_y), + cv::Point((int)res_corners.at(faceId.at(ind_f).at(3) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(3) * 2 + 1) * scale_y), trColors[t.color], 2); } } } float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; for(int k=0; k<8; k++) { - if(res_corners[2*k]bb1) - bb1=res_corners[2*k]; - if(res_corners[2*k+1]bb3) - bb3=res_corners[2*k+1]; + if(res_corners[2*k] < bb0) + bb0 = res_corners[2*k]; + if(res_corners[2*k] > bb1) + bb1 = res_corners[2*k]; + if(res_corners[2*k+1] < bb2) + bb2 = res_corners[2*k+1]; + if(res_corners[2*k+1] > bb3) + bb3 = res_corners[2*k+1]; } // if(not no_bbox): - // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), - // tr_colors[t.color], thickness); - cv::rectangle(frames[bi], cv::Point(bb0, bb2 - text_size.height - thickness), - cv::Point(bb0 + text_size.width, bb2), tr_colors[t.color], -1); + // cv::rectangle(frame, + // cv::Point(bb0, bb2), + // cv::Point(bb1, bb3), + // trColors[t.color], thickness); + cv::rectangle(frames[bi], + cv::Point(bb0 * scale_x, bb2 * scale_y - text_size.height - thickness), + cv::Point(bb0 * scale_x + text_size.width, bb2 * scale_y), + trColors[t.color], -1); - cv::putText(frames[bi], txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, - font_scale, cv::Scalar(255, 255, 255), 1); + cv::putText(frames[bi], txt, + cv::Point(bb0 * scale_x, bb2 * scale_y - thickness -1), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frames[bi], cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), - cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at(0,0)), - (int)((bb2 + bb3)/2 + t.det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); + cv::arrowedLine(frames[bi], + cv::Point((int)((bb0 + bb1)/2) * scale_x, (int)((bb2 + bb3)/2) * scale_y), + cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at(0,0)) * scale_x, + (int)((bb2 + bb3)/2 + t.det_res.tr.at(0,1)) * scale_y), + cv::Scalar(255, 0, 255), 2); } } } From 34c1c3d577cb55235f0c73a5eee201023fc0530b Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 11 May 2021 16:17:23 +0200 Subject: [PATCH 20/33] Update cnet branch. This commit splits the demo3D in two demo: one for the 3D object detection and one for the tracking. It renames the files related to CenterTrack. It adds a new parameter to select the tracker mode (2D or 3D). Signed-off-by: Davide Sapienza --- CMakeLists.txt | 13 +- demo/demo/demo3D.cpp | 5 - demo/demo/demoTracker.cpp | 157 +++++++++++++ ...ternetDetection3DTrack.h => CenterTrack.h} | 20 +- include/tkDNN/TrackingNN.h | 158 +++++++++++++ ...etDetection3DTrack.cpp => CenterTrack.cpp} | 47 ++-- .../dla34_ctrack/dla34_ctrack.cpp} | 222 +++++++++--------- 7 files changed, 469 insertions(+), 153 deletions(-) create mode 100644 demo/demo/demoTracker.cpp rename include/tkDNN/{CenternetDetection3DTrack.h => CenterTrack.h} (90%) create mode 100644 include/tkDNN/TrackingNN.h rename src/{CenternetDetection3DTrack.cpp => CenterTrack.cpp} (96%) rename tests/{centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp => centertrack/dla34_ctrack/dla34_ctrack.cpp} (73%) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb2b1c6..197dced 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,9 +46,9 @@ include_directories(${EIGEN3_INCLUDE_DIR}) find_package(OpenCV REQUIRED) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV") -if(OpenCV_CUDA_VERSION) - add_compile_definitions(OPENCV_CUDACONTRIB) -endif() +# if(OpenCV_CUDA_VERSION) +# add_compile_definitions(OPENCV_CUDACONTRIB) +# endif() # gives problems in cross-compiling, probably malformed cmake config find_package(yaml-cpp REQUIRED) @@ -120,8 +120,8 @@ target_link_libraries(test_resnet101_cnet3d tkDNN) add_executable(test_dla34_cnet3d tests/centernet/dla34_cnet3d/dla34_cnet3d.cpp) target_link_libraries(test_dla34_cnet3d tkDNN) -add_executable(test_dla34_cnet3d_track tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp) -target_link_libraries(test_dla34_cnet3d_track tkDNN) +add_executable(test_dla34_ctrack tests/centertrack/dla34_ctrack/dla34_ctrack.cpp) +target_link_libraries(test_dla34_ctrack tkDNN) # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) @@ -136,6 +136,9 @@ target_link_libraries(demo tkDNN) add_executable(demo3D demo/demo/demo3D.cpp) target_link_libraries(demo3D tkDNN) +add_executable(demoTracker demo/demo/demoTracker.cpp) +target_link_libraries(demoTracker tkDNN) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 620b0d4..90d4bfb 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -5,7 +5,6 @@ #include #include "CenternetDetection3D.h" -#include "CenternetDetection3DTrack.h" bool gRun; bool SAVE_RESULT = false; @@ -55,7 +54,6 @@ int main(int argc, char *argv[]) { SAVE_RESULT = true; tk::dnn::CenternetDetection3D cnet; - tk::dnn::CenternetDetection3DTrack ctrack; tk::dnn::DetectionNN3D *detNN; @@ -64,9 +62,6 @@ int main(int argc, char *argv[]) { case 'c': detNN = &cnet; break; - case 't': - detNN = &ctrack; - break; default: FatalError("Network type not allowed (3rd parameter)\n"); } diff --git a/demo/demo/demoTracker.cpp b/demo/demo/demoTracker.cpp new file mode 100644 index 0000000..ad6e204 --- /dev/null +++ b/demo/demo/demoTracker.cpp @@ -0,0 +1,157 @@ +#include +#include +#include /* srand, rand */ +//#include +#include + +#include "CenterTrack.h" + +bool gRun; +bool SAVE_RESULT = false; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + std::cout<<"detection\n"; + signal(SIGINT, sig_handler); + + + std::string net = "dla34_cnet3d_track_fp32.rt"; + if(argc > 1) + net = argv[1]; + #ifdef __linux__ + std::string input = "../demo/yolo_test.mp4"; + #elif _WIN32 + std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + #endif + + if(argc > 2) + input = argv[2]; + char ntype = 'c'; + if(argc > 3) + ntype = argv[3][0]; + int n_classes = 3; + if(argc > 4) + n_classes = atoi(argv[4]); + int n_batch = 1; + if(argc > 5) + n_batch = atoi(argv[5]); + bool show = true; + if(argc > 6) + show = atoi(argv[6]); + float conf_thresh=0.3; + if(argc > 7) + conf_thresh = atof(argv[7]); + bool t3d = true; + if(argc > 8) + t3d = atoi(argv[8]); + if(n_batch < 1 || n_batch > 64) + FatalError("Batch dim not supported"); + + if(!show) + SAVE_RESULT = true; + + tk::dnn::CenterTrack ctrack; + + tk::dnn::TrackingNN *trackNN; + + switch(ntype) + { + case 'c': + trackNN = &ctrack; + break; + default: + FatalError("Network type not allowed (3rd parameter)\n"); + } + std::vector calibs; + // cv::Mat calib = cv::Mat::zeros(cv::Size(3,3), CV_32F); + // calib.at(0,0) = 864.1243196486207;// * 512.0;//884.081444212;//864.1243196486207 * 512.0;// 633.0; + // calib.at(0,2) = 726.7271690557819;// * 512.0;//0.0;//726.7271690557819 * 512.0;// 0.0; //w/2 + // calib.at(1,1) = 883.6552349216504;// * 512.0;//884.081444212;//883.6552349216504 * 512.0;// 633.0; + // calib.at(1,2) = 506.8548506986564;// * 512.0;//0.0;//506.8548506986564 * 512.0;// 0.0; //h/2 + // calibs.push_back(calib); + // calibs.push_back(calib); + // calibs.push_back(calib); + // calibs.push_back(calib); + trackNN->init(net, n_classes, n_batch, conf_thresh, t3d, calibs); + + gRun = true; + + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + cv::Mat frame; + if(show) + cv::namedWindow("detection", cv::WINDOW_NORMAL); + + std::vector batch_frame; + std::vector batch_dnn_input; + + while(gRun) { + batch_dnn_input.clear(); + batch_frame.clear(); + + for(int bi=0; bi< n_batch; ++bi){ + cap >> frame; + if(!frame.data) + break; + batch_frame.push_back(frame); + + // this will be resized to the net format + batch_dnn_input.push_back(frame.clone()); + } + if(!frame.data) + break; + + //inference + trackNN->update(batch_dnn_input, n_batch, false, nullptr, false); + trackNN->draw(batch_frame); + + if(show){ + for(int bi=0; bi< n_batch; ++bi){ + cv::imshow("detection", batch_frame[bi]); + cv::waitKey(1); + } + } + if(n_batch == 1 && SAVE_RESULT) + resultVideo << frame; + } + + std::cout<<"detection end\n"; + double mean = 0; + + std::cout<pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(trackNN->pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n"; + for(int i=0; ipre_stats.size(); i++) mean += trackNN->pre_stats[i]; mean /= trackNN->pre_stats.size(); + std::cout<<"Avg: "<stats.begin(), trackNN->stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(trackNN->stats.begin(), trackNN->stats.end())<<" ms\n"; + for(int i=0; istats.size(); i++) mean += trackNN->stats[i]; mean /= trackNN->stats.size(); + std::cout<<"Avg: "<post_stats.begin(), trackNN->post_stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(trackNN->post_stats.begin(), trackNN->post_stats.end())<<" ms\n"; + for(int i=0; ipost_stats.size(); i++) mean += trackNN->post_stats[i]; mean /= trackNN->post_stats.size(); + std::cout<<"Avg: "< #include "opencv2/opencv.hpp" @@ -11,7 +11,7 @@ #include // std::iota #include // std::sort -#include "DetectionNN3D.h" +#include "TrackingNN.h" #include "kernelsThrust.h" @@ -49,7 +49,7 @@ struct trackingRes int color; }; -class CenternetDetection3DTrack : public DetectionNN3D +class CenterTrack : public TrackingNN { public: tk::dnn::dataDim_t dim; @@ -133,7 +133,7 @@ public: std::vector> faceId; cv::Scalar trColors[256]; - bool view2d = false; + bool mode3D; //processing struct threshold op; @@ -163,9 +163,11 @@ public: public: tk::dnn::Network *pre_phase_net = nullptr; - CenternetDetection3DTrack() {}; - ~CenternetDetection3DTrack() {}; - bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); + CenterTrack() {}; + ~CenterTrack() {}; + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, + const float conf_thresh=0.3, const bool mode_3d=true, + const std::vector& k_calibs=std::vector()); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); @@ -176,4 +178,4 @@ public: } // namespace tk -#endif /*CENTERNETDETECTION3DTRACK_H*/ \ No newline at end of file +#endif /*CENTERTRACK_H*/ \ No newline at end of file diff --git a/include/tkDNN/TrackingNN.h b/include/tkDNN/TrackingNN.h new file mode 100644 index 0000000..476db53 --- /dev/null +++ b/include/tkDNN/TrackingNN.h @@ -0,0 +1,158 @@ +#ifndef TRACKINGNN_H +#define TRACKINGNN_H + +#include +#include +#include +#ifdef __linux__ +#include +#endif + +#include +#include "utils.h" + +#include +#include +#include + +#include "tkdnn.h" + +// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. + +#ifdef OPENCV_CUDACONTRIB +#include +#include +#endif + + +namespace tk { namespace dnn { + +class TrackingNN { + + protected: + tk::dnn::NetworkRT *netRT = nullptr; + dnnType *input_d; + + std::vector originalSize; + + cv::Scalar colors[256]; + + int nBatches = 1; + +#ifdef OPENCV_CUDACONTRIB + cv::cuda::GpuMat bgr[3]; + cv::cuda::GpuMat imagePreproc; +#else + cv::Mat bgr[3]; + cv::Mat imagePreproc; + dnnType *input; +#endif + + /** + * This method preprocess the image, before feeding it to the NN. + * + * @param frame original frame to adapt for inference. + * @param bi batch index + */ + virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; + + /** + * This method postprocess the output of the NN to obtain the correct + * boundig boxes. + * + * @param bi batch index + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation + */ + virtual void postprocess(const int bi=0,const bool mAP=false) = 0; + + public: + int classes = 0; + float confThreshold = 0.3; /*threshold on the confidence of the boxes*/ + + std::vector pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/ + std::vector classesNames; + + TrackingNN() {}; + ~TrackingNN(){}; + + /** + * Method used to initialize the class, allocate memory and compute + * needed data. + * + * @param tensor_path path to the rt file of the NN. + * @param n_classes number of classes for the given dataset. + * @param n_batches maximum number of batches to use in inference. + * @return true if everything is correct, false otherwise. + */ + virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, + const float conf_thresh=0.3, const bool mode_3d=true, const std::vector& k_calibs=std::vector()) = 0; + + /** + * This method performs the whole detection and tracking of the NN. + * + * @param frames frames to run detection and trcking on. + * @param cur_batches number of batches to use in inference. + * @param save_times if set to true, preprocess, inference and postprocess times + * are saved on a csv file, otherwise not. + * @param times pointer to the output stream where to write times. + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation. + */ + void update(std::vector& frames, const int cur_batches=1, bool save_times=false, + std::ofstream *times=nullptr, const bool mAP=false){ + if(save_times && times==nullptr) + FatalError("save_times set to true, but no valid ofstream given"); + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); + + originalSize.clear(); + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); + { + TKDNN_TSTART + for(int bi=0; biinput_dim; + dim.n = cur_batches; + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + if(save_times) *times<& frames){}; + +}; + +}} + +#endif /* TRACKINGNN_H*/ diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenterTrack.cpp similarity index 96% rename from src/CenternetDetection3DTrack.cpp rename to src/CenterTrack.cpp index d119c1e..dc15823 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenterTrack.cpp @@ -1,16 +1,17 @@ -#include "CenternetDetection3DTrack.h" +#include "CenterTrack.h" namespace tk { namespace dnn { -bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, - const float conf_thresh, const std::vector& k_calibs) { +bool CenterTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, + const float conf_thresh, const bool mode_3d, const std::vector& k_calibs) { netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); dim = netRT->input_dim; dim.c = 3; nBatches = n_batches; confThreshold = conf_thresh; + mode3D = mode_3d; inputCalibs = k_calibs; init_preprocessing(); init_pre_inf(); @@ -18,7 +19,7 @@ bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n init_visualization(n_classes); } -bool CenternetDetection3DTrack::init_preprocessing(){ +bool CenterTrack::init_preprocessing(){ //image transformation src = cv::Mat(cv::Size(2,3), CV_32F); dst = cv::Mat(cv::Size(2,3), CV_32F); @@ -60,12 +61,12 @@ bool CenternetDetection3DTrack::init_preprocessing(){ checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); } -bool CenternetDetection3DTrack::init_pre_inf(){ +bool CenterTrack::init_pre_inf(){ // initial steps: the first part of the network - const char *pre_img_conv1_bin = "dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; - const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; - const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; - const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; + const char *pre_img_conv1_bin = "dla34_ctrack/layers/base-pre_img_layer-0.bin"; + const char *pre_hm_conv1_bin = "dla34_ctrack/layers/base-pre_hm_layer-0.bin"; + const char *conv1_bin = "dla34_ctrack/layers/base-base_layer-0.bin"; + const char *conv2_bin = "dla34_ctrack/layers/base-level0-0.bin"; dim_in0 = tk::dnn::dataDim_t(1, 3, 512, 512, 1); dim_in1 = tk::dnn::dataDim_t(1, 1, 512, 512, 1); @@ -82,9 +83,9 @@ bool CenternetDetection3DTrack::init_pre_inf(){ dnnType *i0_h, *i1_h, *i2_h; // dnnType *i0_d, *i1_d, *i2_d; - // const char *input_bin = "dla34_cnet3d_track/debug/input.bin"; - // const char *pre_img_bin = "dla34_cnet3d_track/debug/pre_imgages.bin"; - // const char *pre_hm_bin = "dla34_cnet3d_track/debug/pre_hms.bin"; + // const char *input_bin = "dla34_ctrack/debug/input.bin"; + // const char *pre_img_bin = "dla34_ctrack/debug/pre_imgages.bin"; + // const char *pre_hm_bin = "dla34_ctrack/debug/pre_hms.bin"; // readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &img_d); // readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &hm_d); // readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &input_pre_inf_d); @@ -114,7 +115,7 @@ bool CenternetDetection3DTrack::init_pre_inf(){ return true; } -bool CenternetDetection3DTrack::init_postprocessing(){ +bool CenterTrack::init_postprocessing(){ srand(0); //seed = 0 for random colors dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1); @@ -203,7 +204,7 @@ bool CenternetDetection3DTrack::init_postprocessing(){ trackId.resize(nBatches, 0); } -bool CenternetDetection3DTrack::init_visualization(const int n_classes){ +bool CenterTrack::init_visualization(const int n_classes){ classes = n_classes; // const char *kitti_class_name[] = { // "person", "car", "bicycle"}; @@ -275,11 +276,11 @@ bool CenternetDetection3DTrack::init_visualization(const int n_classes){ // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } -void CenternetDetection3DTrack::_get_additional_inputs(){ +void CenterTrack::_get_additional_inputs(){ //None no additional input } -void CenternetDetection3DTrack::pre_inf(const int bi){ +void CenterTrack::pre_inf(const int bi){ TKDNN_TSTART tk::dnn::dataDim_t dim_aus; pre_phase_net->infer(dim_aus, nullptr); @@ -289,7 +290,7 @@ void CenternetDetection3DTrack::pre_inf(const int bi){ checkCuda( cudaDeviceSynchronize() ); } -void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ +void CenterTrack::preprocess(cv::Mat &frame, const int bi){ cv::Size sz = originalSize[bi]; // float scale = 1.0; float new_height = dim.h;//sz.height * scale; @@ -403,7 +404,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ checkCuda( cudaDeviceSynchronize() ); } -cv::Mat CenternetDetection3DTrack::transform_preds_with_trans(float x1, float x2){ +cv::Mat CenterTrack::transform_preds_with_trans(float x1, float x2){ cv::Mat target_coords(cv::Size(1,3), CV_32F); target_coords.at(0,0) = x1; target_coords.at(0,1) = x2; @@ -411,7 +412,7 @@ cv::Mat CenternetDetection3DTrack::transform_preds_with_trans(float x1, float x2 return transOut * target_coords; } -void CenternetDetection3DTrack::tracking(const int bi) { +void CenterTrack::tracking(const int bi) { float item_size[countDet]; int item_cl[countDet]; float dets[2*countDet]; @@ -600,7 +601,7 @@ void CenternetDetection3DTrack::tracking(const int bi) { } -void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { +void CenterTrack::postprocess(const int bi, const bool mAP) { dnnType *rt_out[9]; rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi; @@ -734,7 +735,7 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { tracking(bi); } -void CenternetDetection3DTrack::draw(std::vector& frames) { +void CenterTrack::draw(std::vector& frames) { struct trackingRes t; float sc; int id; @@ -755,7 +756,7 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { cv::Size text_size = getTextSize(txt, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); if(t.det_res.score > confThreshold){// && t.active!=0) { - if(view2d) { + if(!mode3D) { cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0) * scale_x, t.det_res.bb0.at(0,1) * scale_y), cv::Point(t.det_res.bb1.at(0,0) * scale_x, t.det_res.bb1.at(0,1) * scale_y), @@ -776,7 +777,7 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { cv::Scalar(255, 0, 255), 2); } //3d - if(!view2d && t.det_res.z > 1){ + if(mode3D && t.det_res.z > 1){ r.at(0,0) = std::cos(t.det_res.rot_y); r.at(0,2) = std::sin(t.det_res.rot_y); r.at(2,0) = -std::sin(t.det_res.rot_y); diff --git a/tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp b/tests/centertrack/dla34_ctrack/dla34_ctrack.cpp similarity index 73% rename from tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp rename to tests/centertrack/dla34_ctrack/dla34_ctrack.cpp index 4829f16..eb3788c 100644 --- a/tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp +++ b/tests/centertrack/dla34_ctrack/dla34_ctrack.cpp @@ -1,130 +1,130 @@ #include #include "tkdnn.h" -const char *input_bin = "dla34_cnet3d_track/debug/input_base-level0-0.bin"; -// const char *input_bin = "dla34_cnet3d_track/debug/input.bin"; -// const char *pre_img_bin = "dla34_cnet3d_track/debug/pre_imgages.bin"; -// const char *pre_hm_bin = "dla34_cnet3d_track/debug/pre_hms.bin"; +const char *input_bin = "dla34_ctrack/debug/input_base-level0-0.bin"; +// const char *input_bin = "dla34_ctrack/debug/input.bin"; +// const char *pre_img_bin = "dla34_ctrack/debug/pre_imgages.bin"; +// const char *pre_hm_bin = "dla34_ctrack/debug/pre_hms.bin"; // //pre -// const char *pre_img_conv1_bin = "dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; -// const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; -// const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; +// const char *pre_img_conv1_bin = "dla34_ctrack/layers/base-pre_img_layer-0.bin"; +// const char *pre_hm_conv1_bin = "dla34_ctrack/layers/base-pre_hm_layer-0.bin"; +// const char *conv1_bin = "dla34_ctrack/layers/base-base_layer-0.bin"; -const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; -const char *conv3_bin = "dla34_cnet3d_track/layers/base-level1-0.bin"; +const char *conv2_bin = "dla34_ctrack/layers/base-level0-0.bin"; +const char *conv3_bin = "dla34_ctrack/layers/base-level1-0.bin"; // s - stage, t - tree -const char *s1_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level2-tree1-conv1.bin"; -const char *s1_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level2-tree1-conv2.bin"; -const char *s1_t1_project = "dla34_cnet3d_track/layers/base-level2-project-0.bin"; -const char *s1_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level2-tree2-conv1.bin"; -const char *s1_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level2-tree2-conv2.bin"; -const char *s1_root_conv1_bin = "dla34_cnet3d_track/layers/base-level2-root-conv.bin"; -const char *s2_t1_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree1-conv1.bin"; -const char *s2_t1_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree1-conv2.bin"; -const char *s2_t1_t1_project = "dla34_cnet3d_track/layers/base-level3-tree1-project-0.bin"; -const char *s2_t1_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree2-conv1.bin"; -const char *s2_t1_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree2-conv2.bin"; -const char *s2_t1_root_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree1-root-conv.bin"; -const char *s2_t2_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree1-conv1.bin"; -const char *s2_t2_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree1-conv2.bin"; -const char *s2_t2_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree2-conv1.bin"; -const char *s2_t2_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree2-conv2.bin"; -const char *s2_t2_root_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree2-root-conv.bin"; -const char *s3_t1_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree1-conv1.bin"; -const char *s3_t1_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree1-conv2.bin"; -const char *s3_t1_t1_project = "dla34_cnet3d_track/layers/base-level4-tree1-project-0.bin"; -const char *s3_t1_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree2-conv1.bin"; -const char *s3_t1_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree2-conv2.bin"; -const char *s3_t1_root_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree1-root-conv.bin"; -const char *s3_t2_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree1-conv1.bin"; -const char *s3_t2_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree1-conv2.bin"; -const char *s3_t2_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree2-conv1.bin"; -const char *s3_t2_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree2-conv2.bin"; -const char *s3_t2_root_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree2-root-conv.bin"; -const char *s4_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level5-tree1-conv1.bin"; -const char *s4_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level5-tree1-conv2.bin"; -const char *s4_t1_project = "dla34_cnet3d_track/layers/base-level5-project-0.bin"; -const char *s4_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level5-tree2-conv1.bin"; -const char *s4_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level5-tree2-conv2.bin"; -const char *s4_root_conv1_bin = "dla34_cnet3d_track/layers/base-level5-root-conv.bin"; +const char *s1_t1_conv1_bin = "dla34_ctrack/layers/base-level2-tree1-conv1.bin"; +const char *s1_t1_conv2_bin = "dla34_ctrack/layers/base-level2-tree1-conv2.bin"; +const char *s1_t1_project = "dla34_ctrack/layers/base-level2-project-0.bin"; +const char *s1_t2_conv1_bin = "dla34_ctrack/layers/base-level2-tree2-conv1.bin"; +const char *s1_t2_conv2_bin = "dla34_ctrack/layers/base-level2-tree2-conv2.bin"; +const char *s1_root_conv1_bin = "dla34_ctrack/layers/base-level2-root-conv.bin"; +const char *s2_t1_t1_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-tree1-conv1.bin"; +const char *s2_t1_t1_conv2_bin = "dla34_ctrack/layers/base-level3-tree1-tree1-conv2.bin"; +const char *s2_t1_t1_project = "dla34_ctrack/layers/base-level3-tree1-project-0.bin"; +const char *s2_t1_t2_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-tree2-conv1.bin"; +const char *s2_t1_t2_conv2_bin = "dla34_ctrack/layers/base-level3-tree1-tree2-conv2.bin"; +const char *s2_t1_root_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-root-conv.bin"; +const char *s2_t2_t1_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-tree1-conv1.bin"; +const char *s2_t2_t1_conv2_bin = "dla34_ctrack/layers/base-level3-tree2-tree1-conv2.bin"; +const char *s2_t2_t2_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-tree2-conv1.bin"; +const char *s2_t2_t2_conv2_bin = "dla34_ctrack/layers/base-level3-tree2-tree2-conv2.bin"; +const char *s2_t2_root_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-root-conv.bin"; +const char *s3_t1_t1_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-tree1-conv1.bin"; +const char *s3_t1_t1_conv2_bin = "dla34_ctrack/layers/base-level4-tree1-tree1-conv2.bin"; +const char *s3_t1_t1_project = "dla34_ctrack/layers/base-level4-tree1-project-0.bin"; +const char *s3_t1_t2_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-tree2-conv1.bin"; +const char *s3_t1_t2_conv2_bin = "dla34_ctrack/layers/base-level4-tree1-tree2-conv2.bin"; +const char *s3_t1_root_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-root-conv.bin"; +const char *s3_t2_t1_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-tree1-conv1.bin"; +const char *s3_t2_t1_conv2_bin = "dla34_ctrack/layers/base-level4-tree2-tree1-conv2.bin"; +const char *s3_t2_t2_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-tree2-conv1.bin"; +const char *s3_t2_t2_conv2_bin = "dla34_ctrack/layers/base-level4-tree2-tree2-conv2.bin"; +const char *s3_t2_root_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-root-conv.bin"; +const char *s4_t1_conv1_bin = "dla34_ctrack/layers/base-level5-tree1-conv1.bin"; +const char *s4_t1_conv2_bin = "dla34_ctrack/layers/base-level5-tree1-conv2.bin"; +const char *s4_t1_project = "dla34_ctrack/layers/base-level5-project-0.bin"; +const char *s4_t2_conv1_bin = "dla34_ctrack/layers/base-level5-tree2-conv1.bin"; +const char *s4_t2_conv2_bin = "dla34_ctrack/layers/base-level5-tree2-conv2.bin"; +const char *s4_root_conv1_bin = "dla34_ctrack/layers/base-level5-root-conv.bin"; //final -// const char *fc_bin = "dla34_cnet3d_track/layers/output.bin"; +// const char *fc_bin = "dla34_ctrack/layers/output.bin"; -const char *ida_0_p_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-proj_1-conv.bin"; -const char *ida_0_p_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin"; -const char *ida_0_up_1_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-up_1.bin"; -const char *ida_0_n_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-node_1-conv.bin"; -const char *ida_0_n_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin"; +const char *ida_0_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_0-proj_1-conv.bin"; +const char *ida_0_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin"; +const char *ida_0_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_0-up_1.bin"; +const char *ida_0_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_0-node_1-conv.bin"; +const char *ida_0_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin"; -const char *ida_1_p_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_1-conv.bin"; -const char *ida_1_p_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin"; -const char *ida_1_up_1_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-up_1.bin"; -const char *ida_1_n_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_1-conv.bin"; -const char *ida_1_n_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin"; -const char *ida_1_p_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_2-conv.bin"; -const char *ida_1_p_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin"; -const char *ida_1_up_2_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-up_2.bin"; -const char *ida_1_n_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_2-conv.bin"; -const char *ida_1_n_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin"; +const char *ida_1_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_1-conv.bin"; +const char *ida_1_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin"; +const char *ida_1_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_1-up_1.bin"; +const char *ida_1_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-node_1-conv.bin"; +const char *ida_1_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin"; +const char *ida_1_p_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_2-conv.bin"; +const char *ida_1_p_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin"; +const char *ida_1_up_2_deconv_bin = "dla34_ctrack/layers/dla_up-ida_1-up_2.bin"; +const char *ida_1_n_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-node_2-conv.bin"; +const char *ida_1_n_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin"; -const char *ida_2_p_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_1-conv.bin"; -const char *ida_2_p_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin"; -const char *ida_2_up_1_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-up_1.bin"; -const char *ida_2_n_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_1-conv.bin"; -const char *ida_2_n_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin"; -const char *ida_2_p_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_2-conv.bin"; -const char *ida_2_p_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin"; -const char *ida_2_up_2_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-up_2.bin"; -const char *ida_2_n_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_2-conv.bin"; -const char *ida_2_n_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin"; -const char *ida_2_p_3_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_3-conv.bin"; -const char *ida_2_p_3_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin"; -const char *ida_2_up_3_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-up_3.bin"; -const char *ida_2_n_3_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_3-conv.bin"; -const char *ida_2_n_3_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin"; +const char *ida_2_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_1-conv.bin"; +const char *ida_2_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin"; +const char *ida_2_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_1.bin"; +const char *ida_2_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_1-conv.bin"; +const char *ida_2_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin"; +const char *ida_2_p_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_2-conv.bin"; +const char *ida_2_p_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin"; +const char *ida_2_up_2_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_2.bin"; +const char *ida_2_n_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_2-conv.bin"; +const char *ida_2_n_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin"; +const char *ida_2_p_3_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_3-conv.bin"; +const char *ida_2_p_3_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin"; +const char *ida_2_up_3_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_3.bin"; +const char *ida_2_n_3_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_3-conv.bin"; +const char *ida_2_n_3_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin"; -const char *ida_up_p_1_dcn_bin = "dla34_cnet3d_track/layers/ida_up-proj_1-conv.bin"; -const char *ida_up_p_1_conv_bin = "dla34_cnet3d_track/layers/ida_up-proj_1-conv-conv_offset_mask.bin"; -const char *ida_up_up_1_deconv_bin = "dla34_cnet3d_track/layers/ida_up-up_1.bin"; -const char *ida_up_n_1_dcn_bin = "dla34_cnet3d_track/layers/ida_up-node_1-conv.bin"; -const char *ida_up_n_1_conv_bin = "dla34_cnet3d_track/layers/ida_up-node_1-conv-conv_offset_mask.bin"; -const char *ida_up_p_2_dcn_bin = "dla34_cnet3d_track/layers/ida_up-proj_2-conv.bin"; -const char *ida_up_p_2_conv_bin = "dla34_cnet3d_track/layers/ida_up-proj_2-conv-conv_offset_mask.bin"; -const char *ida_up_up_2_deconv_bin = "dla34_cnet3d_track/layers/ida_up-up_2.bin"; -const char *ida_up_n_2_dcn_bin = "dla34_cnet3d_track/layers/ida_up-node_2-conv.bin"; -const char *ida_up_n_2_conv_bin = "dla34_cnet3d_track/layers/ida_up-node_2-conv-conv_offset_mask.bin"; +const char *ida_up_p_1_dcn_bin = "dla34_ctrack/layers/ida_up-proj_1-conv.bin"; +const char *ida_up_p_1_conv_bin = "dla34_ctrack/layers/ida_up-proj_1-conv-conv_offset_mask.bin"; +const char *ida_up_up_1_deconv_bin = "dla34_ctrack/layers/ida_up-up_1.bin"; +const char *ida_up_n_1_dcn_bin = "dla34_ctrack/layers/ida_up-node_1-conv.bin"; +const char *ida_up_n_1_conv_bin = "dla34_ctrack/layers/ida_up-node_1-conv-conv_offset_mask.bin"; +const char *ida_up_p_2_dcn_bin = "dla34_ctrack/layers/ida_up-proj_2-conv.bin"; +const char *ida_up_p_2_conv_bin = "dla34_ctrack/layers/ida_up-proj_2-conv-conv_offset_mask.bin"; +const char *ida_up_up_2_deconv_bin = "dla34_ctrack/layers/ida_up-up_2.bin"; +const char *ida_up_n_2_dcn_bin = "dla34_ctrack/layers/ida_up-node_2-conv.bin"; +const char *ida_up_n_2_conv_bin = "dla34_ctrack/layers/ida_up-node_2-conv-conv_offset_mask.bin"; -const char *hm_conv1_bin = "dla34_cnet3d_track/layers/hm-0.bin"; -const char *hm_conv2_bin = "dla34_cnet3d_track/layers/hm-2.bin"; -const char *wh_conv1_bin = "dla34_cnet3d_track/layers/wh-0.bin"; -const char *wh_conv2_bin = "dla34_cnet3d_track/layers/wh-2.bin"; -const char *reg_conv1_bin = "dla34_cnet3d_track/layers/reg-0.bin"; -const char *reg_conv2_bin = "dla34_cnet3d_track/layers/reg-2.bin"; -const char *track_conv1_bin = "dla34_cnet3d_track/layers/tracking-0.bin"; -const char *track_conv2_bin = "dla34_cnet3d_track/layers/tracking-2.bin"; -const char *dep_conv1_bin = "dla34_cnet3d_track/layers/dep-0.bin"; -const char *dep_conv2_bin = "dla34_cnet3d_track/layers/dep-2.bin"; -const char *rot_conv1_bin = "dla34_cnet3d_track/layers/rot-0.bin"; -const char *rot_conv2_bin = "dla34_cnet3d_track/layers/rot-2.bin"; -const char *dim_conv1_bin = "dla34_cnet3d_track/layers/dim-0.bin"; -const char *dim_conv2_bin = "dla34_cnet3d_track/layers/dim-2.bin"; -const char *a_off_conv1_bin = "dla34_cnet3d_track/layers/amodel_offset-0.bin"; -const char *a_off_conv2_bin = "dla34_cnet3d_track/layers/amodel_offset-2.bin"; +const char *hm_conv1_bin = "dla34_ctrack/layers/hm-0.bin"; +const char *hm_conv2_bin = "dla34_ctrack/layers/hm-2.bin"; +const char *wh_conv1_bin = "dla34_ctrack/layers/wh-0.bin"; +const char *wh_conv2_bin = "dla34_ctrack/layers/wh-2.bin"; +const char *reg_conv1_bin = "dla34_ctrack/layers/reg-0.bin"; +const char *reg_conv2_bin = "dla34_ctrack/layers/reg-2.bin"; +const char *track_conv1_bin = "dla34_ctrack/layers/tracking-0.bin"; +const char *track_conv2_bin = "dla34_ctrack/layers/tracking-2.bin"; +const char *dep_conv1_bin = "dla34_ctrack/layers/dep-0.bin"; +const char *dep_conv2_bin = "dla34_ctrack/layers/dep-2.bin"; +const char *rot_conv1_bin = "dla34_ctrack/layers/rot-0.bin"; +const char *rot_conv2_bin = "dla34_ctrack/layers/rot-2.bin"; +const char *dim_conv1_bin = "dla34_ctrack/layers/dim-0.bin"; +const char *dim_conv2_bin = "dla34_ctrack/layers/dim-2.bin"; +const char *a_off_conv1_bin = "dla34_ctrack/layers/amodel_offset-0.bin"; +const char *a_off_conv2_bin = "dla34_ctrack/layers/amodel_offset-2.bin"; const char *output_bin[]={ -"dla34_cnet3d_track/debug/hm.bin", -"dla34_cnet3d_track/debug/wh.bin", -"dla34_cnet3d_track/debug/reg.bin", -"dla34_cnet3d_track/debug/tracking.bin", -"dla34_cnet3d_track/debug/dep.bin", -"dla34_cnet3d_track/debug/rot.bin", -"dla34_cnet3d_track/debug/dim.bin", -"dla34_cnet3d_track/debug/amodel_offset.bin"}; -// const char *output_bin = "dla34_cnet3d_track/debug/base-level0-2.bin"; +"dla34_ctrack/debug/hm.bin", +"dla34_ctrack/debug/wh.bin", +"dla34_ctrack/debug/reg.bin", +"dla34_ctrack/debug/tracking.bin", +"dla34_ctrack/debug/dep.bin", +"dla34_ctrack/debug/rot.bin", +"dla34_ctrack/debug/dim.bin", +"dla34_ctrack/debug/amodel_offset.bin"}; +// const char *output_bin = "dla34_ctrack/debug/base-level0-2.bin"; int main() { - downloadWeightsifDoNotExist("dla34_cnet3d_track/debug/input.bin", "dla34_cnet3d_track", "https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download"); + downloadWeightsifDoNotExist("dla34_ctrack/debug/input.bin", "dla34_ctrack", "https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download"); // Network layout // tk::dnn::dataDim_t dim_in0(1, 3, 512, 512, 1); @@ -570,7 +570,7 @@ int main() net.print(); //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet3d_track")); + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_ctrack")); tk::dnn::dataDim_t dim1 = dim_in0; //input dim printCenteredTitle(" CUDNN inference ", '=', 30); From fd56e64938d2427de88854f519d628f0bc4ae899 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 11 May 2021 17:02:43 +0200 Subject: [PATCH 21/33] Update the README and split it into several files. Signed-off-by: Davide Sapienza --- README.md | 339 +------------------------------------- docs/demo.md | 213 ++++++++++++++++++++++++ docs/exporting_weights.md | 100 +++++++++++ docs/mAP_demo.md | 34 ++++ docs/windows.md | 95 +++++++++++ 5 files changed, 447 insertions(+), 334 deletions(-) create mode 100644 docs/demo.md create mode 100644 docs/exporting_weights.md create mode 100644 docs/mAP_demo.md create mode 100644 docs/windows.md diff --git a/README.md b/README.md index 9e4b811..12f2104 100644 --- a/README.md +++ b/README.md @@ -70,26 +70,11 @@ Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001 - [How to compile this repo](#how-to-compile-this-repo) - [Workflow](#workflow) - [How to export weights](#how-to-export-weights) - - [1)Export weights from darknet](#1export-weights-from-darknet) - - [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101) - - [3)Export weights for CenterNet](#3export-weights-for-centernet) - - [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd) - [Run the demo](#run-the-demo) - - [FP16 inference](#fp16-inference) - - [INT8 inference](#int8-inference) - [mAP demo](#map-demo) - [Existing tests and supported networks](#existing-tests-and-supported-networks) - [References](#references) - [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows-10-experimental) - - [Dependencies-Windows](#dependencies-windows) - - [Compiling tkDNN on Windows](#compiling-tkdnn-on-windows) - - [Run the demo on Windows](#run-the-demo-on-windows) - - [FP16 inference windows](#fp16-inference-windows) - - [INT8 inference windows](#int8-inference-windows) - - [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows) - - - ## Dependencies @@ -126,246 +111,17 @@ Steps needed to do inference on tkDNN with a custom neural network. * Create a new test and define the network, layer by layer using the weights extracted and the output to check the results. * Do inference. -## How to export weights +## Exporting weights -Weights are essential for any network to run inference. For each test a folder organized as follow is needed (in the build folder): -``` - test_nn - |---- layers/ (folder containing a binary file for each layer with the corresponding wieghts and bias) - |---- debug/ (folder containing a binary file for each layer with the corresponding outputs) -``` -Therefore, once the weights have been exported, the folders layers and debug should be placed in the corresponding test. - -### 1)Export weights from darknet -To export weights for NNs that are defined in darknet framework, use [this](https://git.hipert.unimore.it/fgatti/darknet.git) fork of darknet and follow these steps to obtain a correct debug and layers folder, ready for tkDNN. - -``` -git clone https://git.hipert.unimore.it/fgatti/darknet.git -cd darknet -make -mkdir layers debug -./darknet export layers -``` -N.b. Use compilation with CPU (leave GPU=0 in Makefile) if you also want debug. - -### 2)Export weights for DLA34 and ResNet101 -To get weights and outputs needed to run the tests dla34 and resnet101 use the Python script and the Anaconda environment included in the repository. - -Create Anaconda environment and activate it: -``` -conda env create -f file_name.yml -source activate env_name -python