mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

Batch size > 1 for the 3D demo.

Browse Source 
This commit lets to use differtent batch size for 3D CenterNet
and CenterTrack.

Signed-off-by: Davide Sapienza <sapienza.dav@gmail.com>
This commit is contained in:
Davide Sapienza
2021-04-29 11:13:24 +02:00
parent 28fab9c3e1
commit be6ad27c11
6 changed files with 309 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
demo/demo/demo3D.cpp
+47 -24
View File
@@ -1,7 +1,7 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#include <unistd.h>
//#include <unistd.h>
#include <mutex>
#include "CenternetDetection3D.h"
@@ -24,7 +24,12 @@ int main(int argc, char *argv[]) {
std::string net = "dla34_cnet3d_fp32.rt";
if(argc > 1)
net = argv[1];
std::string input = "../demo/yolo_test.mp4";
#ifdef __linux__
std::string input = "../demo/yolo_test.mp4";
#elif _WIN32
std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
#endif
if(argc > 2)
input = argv[2];
char ntype = 'c';
@@ -33,9 +38,18 @@ int main(int argc, char *argv[]) {
int n_classes = 3;
if(argc > 4)
n_classes = atoi(argv[4]);
bool show = false;
int n_batch = 1;
if(argc > 5)
show = atoi(argv[5]);
n_batch = atoi(argv[5]);
bool show = true;
if(argc > 6)
show = atoi(argv[6]);
float conf_thresh=0.3;
if(argc > 7)
conf_thresh = atof(argv[7]);
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
if(!show)
SAVE_RESULT = true;
@@ -57,7 +71,7 @@ int main(int argc, char *argv[]) {
FatalError("Network type not allowed (3rd parameter)\n");
}
detNN->init(net, n_classes);
detNN->init(net, n_classes, n_batch, conf_thresh);
gRun = true;
@@ -75,30 +89,40 @@ int main(int argc, char *argv[]) {
}
cv::Mat frame;
cv::Mat dnn_input;
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
cv::namedWindow("detection", cv::WINDOW_NORMAL);
std::vector<tk::dnn::box> detected_bbox;
std::vector<cv::Mat> batch_frame;
std::vector<cv::Mat> batch_dnn_input;
while(gRun) {
cap >> frame;
if(!frame.data) {
break;
}
// this will be resized to the net format
dnn_input = frame.clone();
batch_dnn_input.clear();
batch_frame.clear();
for(int bi=0; bi< n_batch; ++bi){
cap >> frame;
if(!frame.data)
break;
batch_frame.push_back(frame);
// this will be resized to the net format
batch_dnn_input.push_back(frame.clone());
}
if(!frame.data)
break;
//inference
detNN->update(dnn_input);
frame = detNN->draw(frame);
if(show) {
cv::imshow("detection", frame);
cv::waitKey(1);
}
if(SAVE_RESULT)
detNN->update(batch_dnn_input, n_batch);
detNN->draw(batch_frame);
if(show){
for(int bi=0; bi< n_batch; ++bi){
cv::imshow("detection", batch_frame[bi]);
cv::waitKey(1);
}
}
if(n_batch == 1 && SAVE_RESULT)
resultVideo << frame;
}
@@ -124,7 +148,6 @@ int main(int argc, char *argv[]) {
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
return 0;
}
include/tkDNN/CenternetDetection3D.h
+4 -8
View File
@@ -74,22 +74,18 @@ private:
int *ids_out;
struct threshold op;
float peakThreshold = 0.2;
float centerThreshold = 0.3; //default 0.5
cv::Mat corners, pts3DHomo;
std::vector<box3D> detected3D;
std::vector<int>cls3D;
std::vector<std::vector<int>> face_id;
public:
CenternetDetection3D() {};
~CenternetDetection3D() {};
bool init(const std::string& tensor_path, const int n_classes=3);
void preprocess(cv::Mat &frame);
void postprocess();
cv::Mat draw(cv::Mat &frame);
bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
void draw(std::vector<cv::Mat>& frames);
};
include/tkDNN/CenternetDetection3DTrack.h
+7 -6
View File
@@ -145,6 +145,7 @@ private:
int count_det;
//tracks
std::vector<struct trackingRes> tr_res;
std::vector<std::vector<struct trackingRes>> batchTracked;
int count_tr;
int track_id=0;
@@ -153,7 +154,7 @@ private:
bool init_pre_inf();
bool init_postprocessing();
bool init_visualization(const int n_classes);
void pre_inf();
void pre_inf(const int bi);
void _get_additional_inputs();
cv::Mat transform_preds_with_trans(float x1, float x2);
void tracking();
@@ -161,11 +162,11 @@ private:
public:
tk::dnn::Network *pre_phase_net = nullptr;
CenternetDetection3DTrack() {};
~CenternetDetection3DTrack() {};
bool init(const std::string& tensor_path, const int n_classes=3);
void preprocess(cv::Mat &frame);
void postprocess();
cv::Mat draw(cv::Mat &frame);
~CenternetDetection3DTrack() {};
bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
void draw(std::vector<cv::Mat>& frames);
};
include/tkDNN/DetectionNN3D.h
+52 -31
View File
@@ -3,8 +3,11 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
@@ -30,10 +33,12 @@ class DetectionNN3D {
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_d;
cv::Size originalSize;
std::vector<cv::Size> originalSize;
cv::Scalar colors[256];
int nBatches = 1;
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat bgr[3];
cv::cuda::GpuMat imagePreproc;
@@ -47,21 +52,26 @@ class DetectionNN3D {
* This method preprocess the image, before feeding it to the NN.
*
* @param frame original frame to adapt for inference.
* @param bi batch index
*/
virtual void preprocess(cv::Mat &frame) = 0;
virtual void preprocess(cv::Mat &frame, const int bi=0) = 0;
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param bi batch index
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
virtual void postprocess() = 0;
virtual void postprocess(const int bi=0,const bool mAP=false) = 0;
public:
int classes = 0;
float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
std::vector<tk::dnn::box> detected; /*bounding boxes in output*/
std::vector<tk::dnn::box3D> detected3D; /*bounding boxes in output*/
std::vector<std::vector<tk::dnn::box3D>> batchDetected; /*bounding boxes in output*/
std::vector<double> pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/
std::vector<std::string> classesNames;
@@ -69,68 +79,79 @@ class DetectionNN3D {
~DetectionNN3D(){};
/**
* Method used to inialize the class, allocate memory and compute
* Method used to initialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file og the NN.
* @param tensor_path path to the rt file of the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference.
* @return true if everything is correct, false otherwise.
*/
virtual bool init(const std::string& tensor_path, const int n_classes=3) = 0;
/**
* Method to draw boundixg boxes and labels on a frame.
*
* @param frame orginal frame to draw bounding box on.
* @return frame with boundig boxes.
*/
virtual cv::Mat draw(cv::Mat &frame){};
virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3) = 0;
/**
* This method performs the whole detection of the NN.
*
* @param frame frame to run detection on.
* @param frames frames to run detection on.
* @param cur_batches number of batches to use in inference.
* @param save_times if set to true, preprocess, inference and postprocess times
* are saved on a csv file, otherwise not.
* @param times pointer to the output stream where to write times
* @param times pointer to the output stream where to write times.
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation.
*/
void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){
if(!frame.data)
FatalError("No image data feed to detection");
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){
if(save_times && times==nullptr)
FatalError("save_times set to true, but no valid ofstream given");
if(cur_batches > nBatches)
FatalError("A batch size greater than nBatches cannot be used");
originalSize = frame.size();
printCenteredTitle(" TENSORRT detection ", '=', 30);
originalSize.clear();
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
{
TKDNN_TSTART
preprocess(frame);
for(int bi=0; bi<cur_batches;++bi){
if(!frames[bi].data)
FatalError("No image data feed to detection");
originalSize.push_back(frames[bi].size());
preprocess(frames[bi], bi);
}
TKDNN_TSTOP
pre_stats.push_back(t_ns);
pre_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
//do inference
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = cur_batches;
{
dim.print();
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
dim.print();
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
batchDetected.clear();
{
TKDNN_TSTART
postprocess();
for(int bi=0; bi<cur_batches;++bi)
postprocess(bi, mAP);
TKDNN_TSTOP
post_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<"\n";
}
}
}
/**
* Method to draw bounding boxes and labels on a frame.
*
* @param frames original frame to draw bounding box on.
*/
virtual void draw(std::vector<cv::Mat>& frames){};
};
}}
src/CenternetDetection3D.cpp
+52 -49
View File
@@ -3,10 +3,12 @@
namespace tk { namespace dnn {
bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes){
bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) {
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
classes = n_classes;
nBatches = n_batches;
confThreshold = conf_thresh;
dim = netRT->input_dim;
@@ -28,7 +30,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas
trans = cv::Mat(cv::Size(3,2), CV_32F);
trans2 = cv::Mat(cv::Size(3,2), CV_32F);
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot()));
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
dim_hm = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
@@ -91,7 +93,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas
checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
#else
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot()));
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
mean << 0.485, 0.456, 0.406;
stddev << 0.229, 0.224, 0.225;
#endif
@@ -154,13 +156,13 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas
// ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]);
}
void CenternetDetection3D::preprocess(cv::Mat &frame){
void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){
// -----------------------------------pre-process ------------------------------------------
// auto start_t = std::chrono::steady_clock::now();
// auto step_t = std::chrono::steady_clock::now();
// auto end_t = std::chrono::steady_clock::now();
cv::Size sz = originalSize;
cv::Size sz = originalSize[bi];
// std::cout<<"image: "<<sz.width<<", "<<sz.height<<std::endl;
cv::Size sz_old;
float scale = 1.0;
@@ -238,7 +240,7 @@ void CenternetDetection3D::preprocess(cv::Mat &frame){
// std::cout << " TIME normalize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
checkCuda(cudaMemcpy(input_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
@@ -280,21 +282,21 @@ void CenternetDetection3D::preprocess(cv::Mat &frame){
int idx = i*imageF.rows*imageF.cols;
int ch = dim2.c-3 +i;
// std::cout<<"i: "<<i<<", idx: "<<idx<<", ch: "<<ch<<std::endl;
memcpy((void*)&input[idx], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
memcpy((void*)&input[idx+ netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
#endif
}
void CenternetDetection3D::postprocess(){
void CenternetDetection3D::postprocess(const int bi, const bool mAP) {
dnnType *rt_out[7];
rt_out[0] = (dnnType *)netRT->buffersRT[1];
rt_out[1] = (dnnType *)netRT->buffersRT[2];
rt_out[2] = (dnnType *)netRT->buffersRT[3];
rt_out[3] = (dnnType *)netRT->buffersRT[4];
rt_out[4] = (dnnType *)netRT->buffersRT[5];
rt_out[5] = (dnnType *)netRT->buffersRT[6];
rt_out[6] = (dnnType *)netRT->buffersRT[7];
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi;
rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi;
rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi;
// ------------------------------------ process --------------------------------------------
activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot());
@@ -404,8 +406,7 @@ void CenternetDetection3D::postprocess(){
if(rot_y<M_PI)
rot_y += 2*M_PI;
// if(scores[j] > peakThreshold) {
if(scores[j] > centerThreshold) {
if(scores[j] > confThreshold) {
if(z>0) {
// compute_box_3d
r.at<float>(0,0) = std::cos(rot_y);
@@ -457,16 +458,17 @@ void CenternetDetection3D::postprocess(){
}
res.cl = i;
res.prob = scores[j];
res.print();
//res.print();
detected3D.push_back(res);
}
}
}
}
}
batchDetected.push_back(detected3D);
}
cv::Mat CenternetDetection3D::draw(cv::Mat &frame) {
void CenternetDetection3D::draw(std::vector<cv::Mat>& frames) {
tk::dnn::box3D b;
int x0, w, x1, y0, h, y1;
int objClass;
@@ -476,40 +478,41 @@ cv::Mat CenternetDetection3D::draw(cv::Mat &frame) {
float font_scale = 0.5;
int thickness = 2;
// draw dets
for(int i=0; i<detected3D.size(); i++) {
b = detected3D[i];
for(int bi=0; bi<frames.size(); ++bi){
// draw dets
for(int i=0; i<batchDetected[bi].size(); i++) {
b = batchDetected[bi][i];
for(int ind_f = 3; ind_f>=0; ind_f--) {
for(int j=0; j<4; j++) {
cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2),
b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)),
cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2),
b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)),
colors[b.cl], 2);
if(ind_f == 0) {
cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2),
b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)),
cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2),
b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2);
cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2),
b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)),
cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2),
b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2);
for(int ind_f = 3; ind_f>=0; ind_f--) {
for(int j=0; j<4; j++) {
cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2),
b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)),
cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2),
b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)),
colors[b.cl], 2);
if(ind_f == 0) {
cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2),
b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)),
cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2),
b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2);
cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2),
b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)),
cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2),
b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2);
}
}
}
// draw label
cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
cv::rectangle(frames[bi], cv::Point(b.corners.at(face_id.at(0).at(0) * 2),
b.corners.at(face_id.at(0).at(0) * 2 + 1)),
cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2),
(b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1);
cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2),
b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness);
}
// draw label
cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
cv::rectangle(frame, cv::Point(b.corners.at(face_id.at(0).at(0) * 2),
b.corners.at(face_id.at(0).at(0) * 2 + 1)),
cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2),
(b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1);
cv::putText(frame, classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2),
b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness);
}
return frame;
}
}}
src/CenternetDetection3DTrack.cpp
+147 -144
View File
@@ -3,12 +3,14 @@
namespace tk { namespace dnn {
bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes){
std::cout<<(tensor_path).c_str()<<"\n";
bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) {
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
dim = netRT->input_dim;
dim.c = 3;
nBatches = n_batches;
confThreshold = conf_thresh;
init_preprocessing();
init_pre_inf();
@@ -46,13 +48,13 @@ bool CenternetDetection3DTrack::init_preprocessing(){
checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
#else
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot()));
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot() * nBatches));
mean << 0.40789655, 0.44719303, 0.47026116;
stddev << 0.2886383, 0.27408165, 0.27809834;
#endif
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot()));
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
checkCuda(cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot()));
checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) );
}
@@ -276,20 +278,20 @@ void CenternetDetection3DTrack::_get_additional_inputs(){
//None no additional input
}
void CenternetDetection3DTrack::pre_inf(){
void CenternetDetection3DTrack::pre_inf(const int bi){
TKDNN_TSTART
tk::dnn::dataDim_t dim_aus;
pre_phase_net->infer(dim_aus, nullptr);
TKDNN_TSTOP
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(input_d, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
checkCuda( cudaDeviceSynchronize() );
}
void CenternetDetection3DTrack::preprocess(cv::Mat &frame){
// -----------------------------------pre-process ------------------------------------------
cv::Size sz = originalSize;
void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){
// -----------------------------------pre-process ------------------------------------------
batchTracked.clear();
cv::Size sz = originalSize[bi];
cv::Size sz_old;
float scale = 1.0;
float new_height = sz.height * scale;
@@ -302,7 +304,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame){
// float s = new_width >= new_height ? new_width : new_height;
// ----------- get_affine_transform
// rot_rad = pi * 0 / 100 --> 0
dim.print();
//dim.print();
src.at<float>(0,0)=c[0];
src.at<float>(0,1)=c[1];
src.at<float>(1,0)=c[0];
@@ -389,7 +391,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame){
checkCuda( cudaDeviceSynchronize() );
iter0=false;
}
pre_inf();
pre_inf(bi);
checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
checkCuda( cudaDeviceSynchronize() );
@@ -587,17 +589,17 @@ void CenternetDetection3DTrack::tracking(){
}
void CenternetDetection3DTrack::postprocess(){
void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) {
dnnType *rt_out[9];
rt_out[0] = (dnnType *)netRT->buffersRT[1];
rt_out[1] = (dnnType *)netRT->buffersRT[2];
rt_out[2] = (dnnType *)netRT->buffersRT[3];
rt_out[3] = (dnnType *)netRT->buffersRT[4];
rt_out[4] = (dnnType *)netRT->buffersRT[5];
rt_out[5] = (dnnType *)netRT->buffersRT[6];
rt_out[6] = (dnnType *)netRT->buffersRT[7];
rt_out[7] = (dnnType *)netRT->buffersRT[8];
rt_out[8] = (dnnType *)netRT->buffersRT[9];
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi;
rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi;
rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi;
rt_out[7] = (dnnType *)netRT->buffersRT[8]+ netRT->buffersDIM[8].tot()*bi;
rt_out[8] = (dnnType *)netRT->buffersRT[9]+ netRT->buffersDIM[9].tot()*bi;
// ------------------------------------ process --------------------------------------------
@@ -719,143 +721,144 @@ void CenternetDetection3DTrack::postprocess(){
}
// track step
tracking();
batchTracked.push_back(tr_res);
}
cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) {
void CenternetDetection3DTrack::draw(std::vector<cv::Mat>& frames) {
struct trackingRes t;
float sc;
int id;
std::string txt;
int baseline = 0;
float font_scale = 0.8;
int thickness = 2;
for(int i=0; i<count_tr; i++) {
id = tr_res[i].tracking_id;
txt = classesNames[tr_res[i].det_res.cl-1]+'-'+std::to_string(id); //forse ha bisogno di cl-1
cv::Size text_size = getTextSize(txt, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
if(tr_res[i].det_res.score > vis_thresh){// && tr_res[i].active!=0) {
if(view2d) {
cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at<float>(0,0), tr_res[i].det_res.bb0.at<float>(0,1)),
cv::Point(tr_res[i].det_res.bb1.at<float>(0,0), tr_res[i].det_res.bb1.at<float>(0,1)), tr_colors[tr_res[i].color], thickness);
cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at<float>(0,0),
tr_res[i].det_res.bb0.at<float>(0,1) - text_size.height - thickness),
cv::Point(tr_res[i].det_res.bb0.at<float>(0,0) + text_size.width,
tr_res[i].det_res.bb0.at<float>(0,1)), tr_colors[tr_res[i].color], -1);
cv::putText(frame, txt, cv::Point(tr_res[i].det_res.bb0.at<float>(0,0),
tr_res[i].det_res.bb0.at<float>(0,1) - thickness -1),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1);
int thickness = 2;
for(int bi=0; bi<frames.size(); ++bi) {
// draw dets
for(int i=0; i<batchTracked[bi].size(); i++) {
t = batchTracked[bi][i];
id = t.tracking_id;
txt = classesNames[t.det_res.cl-1]+'-'+std::to_string(id); //forse ha bisogno di cl-1
cv::Size text_size = getTextSize(txt, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
if(t.det_res.score > vis_thresh){// && t.active!=0) {
if(view2d) {
cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at<float>(0,0), t.det_res.bb0.at<float>(0,1)),
cv::Point(t.det_res.bb1.at<float>(0,0), t.det_res.bb1.at<float>(0,1)), tr_colors[t.color], thickness);
cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at<float>(0,0),
t.det_res.bb0.at<float>(0,1) - text_size.height - thickness),
cv::Point(t.det_res.bb0.at<float>(0,0) + text_size.width,
t.det_res.bb0.at<float>(0,1)), tr_colors[t.color], -1);
cv::putText(frames[bi], txt, cv::Point(t.det_res.bb0.at<float>(0,0),
t.det_res.bb0.at<float>(0,1) - thickness -1),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1);
cv::arrowedLine(frame, cv::Point((int)tr_res[i].det_res.ct.at<float>(0,0),
(int)tr_res[i].det_res.ct.at<float>(0,1)),
cv::Point((int)(tr_res[i].det_res.ct.at<float>(0,0) + tr_res[i].det_res.tr.at<float>(0,0)),
(int)(tr_res[i].det_res.ct.at<float>(0,1) + tr_res[i].det_res.tr.at<float>(0,1))),
cv::Scalar(255, 0, 255), 2);
}
//3d
if(!view2d && tr_res[i].det_res.z > 1){
r.at<float>(0,0) = std::cos(tr_res[i].det_res.rot_y);
r.at<float>(0,2) = std::sin(tr_res[i].det_res.rot_y);
r.at<float>(2,0) = -std::sin(tr_res[i].det_res.rot_y);
r.at<float>(2,2) = std::cos(tr_res[i].det_res.rot_y);
corners.at<float>(0,0) = tr_res[i].det_res.dim[2]/2;
corners.at<float>(0,1) = tr_res[i].det_res.dim[2]/2;
corners.at<float>(0,2) = -tr_res[i].det_res.dim[2]/2;
corners.at<float>(0,3) = -tr_res[i].det_res.dim[2]/2;
corners.at<float>(0,4) = tr_res[i].det_res.dim[2]/2;
corners.at<float>(0,5) = tr_res[i].det_res.dim[2]/2;
corners.at<float>(0,6) = -tr_res[i].det_res.dim[2]/2;
corners.at<float>(0,7) = -tr_res[i].det_res.dim[2]/2;
corners.at<float>(1,4) = -tr_res[i].det_res.dim[0];
corners.at<float>(1,5) = -tr_res[i].det_res.dim[0];
corners.at<float>(1,6) = -tr_res[i].det_res.dim[0];
corners.at<float>(1,7) = -tr_res[i].det_res.dim[0];
corners.at<float>(2,0) = tr_res[i].det_res.dim[1]/2;
corners.at<float>(2,1) = -tr_res[i].det_res.dim[1]/2;
corners.at<float>(2,2) = -tr_res[i].det_res.dim[1]/2;
corners.at<float>(2,3) = tr_res[i].det_res.dim[1]/2;
corners.at<float>(2,4) = tr_res[i].det_res.dim[1]/2;
corners.at<float>(2,5) = -tr_res[i].det_res.dim[1]/2;
corners.at<float>(2,6) = -tr_res[i].det_res.dim[1]/2;
corners.at<float>(2,7) = tr_res[i].det_res.dim[1]/2;
cv::Mat aus = r * corners;
for(int k=0; k<8; k++) {
aus.at<float>(0,k) += tr_res[i].det_res.x;
aus.at<float>(1,k) += tr_res[i].det_res.y;
aus.at<float>(2,k) += tr_res[i].det_res.z;
cv::arrowedLine(frames[bi], cv::Point((int)t.det_res.ct.at<float>(0,0),
(int)t.det_res.ct.at<float>(0,1)),
cv::Point((int)(t.det_res.ct.at<float>(0,0) + t.det_res.tr.at<float>(0,0)),
(int)(t.det_res.ct.at<float>(0,1) + t.det_res.tr.at<float>(0,1))),
cv::Scalar(255, 0, 255), 2);
}
// corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3)));
for(int k1=0; k1<3; k1++) {
for(int k2=0; k2<8; k2++)
pts3DHomo.at<float>(k1,k2) = aus.at<float>(k1,k2);
}
aus.release();
aus = calibs * pts3DHomo;
std::vector<float> res_corners;
for(int k=0; k<8; k++) {
res_corners.push_back(aus.at<float>(0,k) / aus.at<float>(2,k));
res_corners.push_back(aus.at<float>(1,k) / aus.at<float>(2,k));
}
aus.release();
for(int ind_f = 3; ind_f>=0; ind_f--) {
for(int j=0; j<4; j++) {
cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2),
(int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)),
cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2),
(int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)),
tr_colors[tr_res[i].color], 2);
if(ind_f == 0 && j==3) {
cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2),
(int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)),
cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2),
(int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2);
cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2),
(int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)),
cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2),
(int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2);
//3d
if(!view2d && t.det_res.z > 1){
r.at<float>(0,0) = std::cos(t.det_res.rot_y);
r.at<float>(0,2) = std::sin(t.det_res.rot_y);
r.at<float>(2,0) = -std::sin(t.det_res.rot_y);
r.at<float>(2,2) = std::cos(t.det_res.rot_y);
corners.at<float>(0,0) = t.det_res.dim[2]/2;
corners.at<float>(0,1) = t.det_res.dim[2]/2;
corners.at<float>(0,2) = -t.det_res.dim[2]/2;
corners.at<float>(0,3) = -t.det_res.dim[2]/2;
corners.at<float>(0,4) = t.det_res.dim[2]/2;
corners.at<float>(0,5) = t.det_res.dim[2]/2;
corners.at<float>(0,6) = -t.det_res.dim[2]/2;
corners.at<float>(0,7) = -t.det_res.dim[2]/2;
corners.at<float>(1,4) = -t.det_res.dim[0];
corners.at<float>(1,5) = -t.det_res.dim[0];
corners.at<float>(1,6) = -t.det_res.dim[0];
corners.at<float>(1,7) = -t.det_res.dim[0];
corners.at<float>(2,0) = t.det_res.dim[1]/2;
corners.at<float>(2,1) = -t.det_res.dim[1]/2;
corners.at<float>(2,2) = -t.det_res.dim[1]/2;
corners.at<float>(2,3) = t.det_res.dim[1]/2;
corners.at<float>(2,4) = t.det_res.dim[1]/2;
corners.at<float>(2,5) = -t.det_res.dim[1]/2;
corners.at<float>(2,6) = -t.det_res.dim[1]/2;
corners.at<float>(2,7) = t.det_res.dim[1]/2;
cv::Mat aus = r * corners;
for(int k=0; k<8; k++) {
aus.at<float>(0,k) += t.det_res.x;
aus.at<float>(1,k) += t.det_res.y;
aus.at<float>(2,k) += t.det_res.z;
}
// corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3)));
for(int k1=0; k1<3; k1++) {
for(int k2=0; k2<8; k2++)
pts3DHomo.at<float>(k1,k2) = aus.at<float>(k1,k2);
}
aus.release();
aus = calibs * pts3DHomo;
std::vector<float> res_corners;
for(int k=0; k<8; k++) {
res_corners.push_back(aus.at<float>(0,k) / aus.at<float>(2,k));
res_corners.push_back(aus.at<float>(1,k) / aus.at<float>(2,k));
}
aus.release();
for(int ind_f = 3; ind_f>=0; ind_f--) {
for(int j=0; j<4; j++) {
cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2),
(int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)),
cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2),
(int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)),
tr_colors[t.color], 2);
if(ind_f == 0 && j==3) {
cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2),
(int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)),
cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2),
(int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[t.color], 2);
cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2),
(int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)),
cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2),
(int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[t.color], 2);
}
}
}
}
float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0;
for(int k=0; k<8; k++) {
if(res_corners[2*k]<bb0)
bb0=res_corners[2*k];
if(res_corners[2*k]>bb1)
bb1=res_corners[2*k];
if(res_corners[2*k+1]<bb2)
bb2=res_corners[2*k+1];
if(res_corners[2*k+1]>bb3)
bb3=res_corners[2*k+1];
}
// if(not no_bbox):
// cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3),
// tr_colors[tr_res[i].color], thickness);
cv::rectangle(frame, cv::Point(bb0, bb2 - text_size.height - thickness),
cv::Point(bb0 + text_size.width, bb2), tr_colors[tr_res[i].color], -1);
cv::putText(frame, txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX,
font_scale, cv::Scalar(255, 255, 255), 1);
float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0;
for(int k=0; k<8; k++) {
if(res_corners[2*k]<bb0)
bb0=res_corners[2*k];
if(res_corners[2*k]>bb1)
bb1=res_corners[2*k];
if(res_corners[2*k+1]<bb2)
bb2=res_corners[2*k+1];
if(res_corners[2*k+1]>bb3)
bb3=res_corners[2*k+1];
}
// if(not no_bbox):
// cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3),
// tr_colors[t.color], thickness);
cv::rectangle(frames[bi], cv::Point(bb0, bb2 - text_size.height - thickness),
cv::Point(bb0 + text_size.width, bb2), tr_colors[t.color], -1);
cv::putText(frames[bi], txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX,
font_scale, cv::Scalar(255, 255, 255), 1);
cv::arrowedLine(frame, cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)),
cv::Point((int)((bb0 + bb1)/2 + tr_res[i].det_res.tr.at<float>(0,0)),
(int)((bb2 + bb3)/2 + tr_res[i].det_res.tr.at<float>(0,1))),
cv::Scalar(255, 0, 255), 2);
cv::arrowedLine(frames[bi], cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)),
cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at<float>(0,0)),
(int)((bb2 + bb3)/2 + t.det_res.tr.at<float>(0,1))),
cv::Scalar(255, 0, 255), 2);
}
}
}
}
return frame;
}
}}