mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

Refactoring for detection NN

Browse Source 
Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>
This commit is contained in:
Micaela Verucchi
2020-03-20 21:14:12 +01:00
parent c7d9c38ea0
commit bbcc33c0cf
11 changed files with 637 additions and 678 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/tkDNN/CenternetDetection.h
+68 -112
View File
@@ -1,134 +1,90 @@
#include <iostream>
#include <cstring>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#include <unistd.h>
#include <mutex>
#include "utils.h"
#include <time.h>
#ifndef CENTERNETDETECTION_H
#define CENTERNETDETECTION_H
#include "kernels.h"
#include <opencv2/videoio.hpp>
#include "opencv2/opencv.hpp"
#include <time.h>
#include <vector>
#include <numeric> // std::iota
#include <algorithm> // std::sort
#include "DetectionNN.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "opencv2/opencv.hpp"
#include "tkdnn.h"
#include "sorting.h"
namespace tk { namespace dnn {
/**
*
* @author Francesco Gatti
*/
class CenternetDetection {
namespace tk { namespace dnn {
private:
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_d;
class CenternetDetection : public DetectionNN
{
private:
std::vector<std::string> classesNames;
int ndets = 0;
// tk::dnn::Yolo::detection *dets = nullptr;
tk::dnn::dataDim_t dim;
tk::dnn::dataDim_t dim2;
tk::dnn::dataDim_t dim_hm;
tk::dnn::dataDim_t dim_wh;
tk::dnn::dataDim_t dim_reg;
float *topk_scores;
int *topk_inds_;
float *topk_ys_;
float *topk_xs_;
int *ids_d, *ids_, *ids_2, *ids_2d;
cv::Mat imageOrig;
// std::vector< cv::cuda::GpuMat > bgr;
float *scores, *scores_d;
int *clses, *clses_d;
int *topk_inds_d;
float *topk_ys_d;
float *topk_xs_d;
int *inttopk_xs_d, *inttopk_ys_d;
// variable to test cnet on dog pictures
tk::dnn::dataDim_t dim;
tk::dnn::dataDim_t dim2;
cv::Size sz, sz_old;
const char *input_bin = "../tests/resnet101_cnet/debug/input.bin";
cv::cuda::Stream stream;
struct threshold op;
// pre-process
tk::dnn::dataDim_t dim_hm;
tk::dnn::dataDim_t dim_wh;
tk::dnn::dataDim_t dim_reg;
float *topk_scores;
int *topk_inds_;
float *topk_ys_;
float *topk_xs_;
int *ids_d, *ids_, *ids_2, *ids_2d;
float *scores, *scores_d;
int *clses, *clses_d;
int *topk_inds_d;
float *topk_ys_d;
float *topk_xs_d;
int *inttopk_xs_d, *inttopk_ys_d;
float *bbx0, *bby0, *bbx1, *bby1;
float *bbx0_d, *bby0_d, *bbx1_d, *bby1_d;
float *target_coords;
float *bbx0, *bby0, *bbx1, *bby1;
float *bbx0_d, *bby0_d, *bbx1_d, *bby1_d;
float *target_coords;
#ifdef OPENCV_CUDA
float *mean_d;
float *stddev_d;
#else
cv::Vec<float, 3> mean;
cv::Vec<float, 3> stddev;
dnnType *input;
#endif
#ifdef OPENCV_CUDA
float *mean_d;
float *stddev_d;
#else
cv::Vec<float, 3> mean;
cv::Vec<float, 3> stddev;
dnnType *input;
#endif
float *d_ptrs;
float *d_ptrs;
cv::Mat src;
cv::Mat dst;
cv::Mat dst2;
cv::Mat trans, trans2;
//processing
float toll = 0.000001;
int K = 100;
int width = 128;//56; // TODO
cv::Mat src;
cv::Mat dst;
cv::Mat dst2;
cv::Mat trans, trans2;
//processing
float toll = 0.000001;
int K = 100;
int width = 128;//56; // TODO
// pointer used in the kernels
float *src_out;
int *ids_out;
struct threshold op;
// pointer used in the kernels
float *src_out;
int *ids_out;
void preprocess();
public:
dnnType *rt_out[4];
float inp_height = 512;//224;//512;
float inp_width = 512;//224;//512;
int classes = 80;
int num = 0;
int n_masks = 0;
float thresh = 0.3;
cv::Scalar colors[256];
// this is filled with results
std::vector<tk::dnn::box> detected;
// draw
std::vector<std::string> coco_class_name;
// keep track of inference times (ms)
std::vector<double> stats;
CenternetDetection() {}
virtual ~CenternetDetection() {}
/**
* Method used for inizialize the class
*
* @return Success of the initialization
*/
bool init(std::string tensor_path);
cv::Mat draw(cv::Mat &frame);
void update(cv::Mat &frame);
public:
CenternetDetection() {};
~CenternetDetection() {};
bool init(const std::string& tensor_path, const int n_classes=80);
void preprocess(cv::Mat &frame);
void update(cv::Mat &frame);
void postprocess(dnnType **rt_out, const int n_out);
cv::Mat draw(cv::Mat &frame);
};
}}
} // namespace dnn
} // namespace tk
#endif /*CENTERNETDETECTION_H*/
include/tkDNN/DetectionNN.h
+99
View File
@@ -0,0 +1,99 @@
#ifndef DETECTIONNN_H
#define DETECTIONNN_H
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#include <unistd.h>
#include <mutex>
#include "utils.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "tkdnn.h"
// #define OPENCV_CUDA //if OPENCV has been compiled with CUDA and contrib.
namespace tk { namespace dnn {
enum networkType_t{
NETWORK_YOLO3,
NETWORK_MOBILENETSSDLITE,
NETWORK_CENTERNET
};
class DetectionNN {
protected:
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_d;
cv::Size originalSize;
cv::Scalar colors[256];
#ifdef OPENCV_CUDA
cv::cuda::GpuMat bgr[3];
cv::cuda::GpuMat imagePreproc;
#else
cv::Mat bgr[3];
cv::Mat imagePreproc;
dnnType *input;
#endif
public:
int classes = 0;
float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
std::vector<tk::dnn::box> detected; /*bounding boxes in output*/
std::vector<double> stats; /*keeps track of inference times (ms)*/
DetectionNN() {};
~DetectionNN(){};
/**
* Method used to inialize the class, allocate memory and compute
* needed data.
*
* @param path to the rt file og the NN.
* @return true if everything is correct, false otherwise.
*/
virtual bool init(const std::string& tensor_path, const int n_classes=80) = 0;
/**
* This method preprocess the image, before feeding it to the NN.
*
* @param original frame to adapt for inference.
*/
virtual void preprocess(cv::Mat &frame) = 0;
/**
* This method performs the inference of the NN.
*
* @param frame to run inference on.
*/
virtual void update(cv::Mat &frame) = 0;
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param outputs of the inference
* @param number of outputs of the inference
*/
virtual void postprocess(dnnType **rt_out, const int n_out) = 0;
/**
* Method to draw boundixg boxes and labels on a frame.
*
* @param orginal frame to draw bounding box on.
* @return frame with boundig boxes.
*/
virtual cv::Mat draw(cv::Mat &frame) = 0;
};
}}
#endif /* DETECTIONNN_H*/
include/tkDNN/MobilenetDetection.h
+20 -51
View File
@@ -1,18 +1,15 @@
#ifndef MOBILENETDETECTION_H
#define MOBILENETDETECTION_H
#include <iostream>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "opencv2/opencv.hpp"
#include "tkdnn.h"
#include "DetectionNN.h"
#define N_COORDS 4
#define N_SSDSPEC 6
namespace tk { namespace dnn {
struct SSDSpec
{
@@ -24,10 +21,9 @@ struct SSDSpec
int ratio2 = 0;
SSDSpec() {}
SSDSpec(int feature_size, int shrinkage, int box_width, int box_height, int ratio1, int ratio2) : featureSize(feature_size), shrinkage(shrinkage), boxWidth(box_width), boxHeight(box_height),
ratio1(ratio1), ratio2(ratio2) {}
SSDSpec(int feature_size, int shrinkage, int box_width, int box_height, int ratio1, int ratio2) :
featureSize(feature_size), shrinkage(shrinkage), boxWidth(box_width),
boxHeight(box_height), ratio1(ratio1), ratio2(ratio2) {}
void setAll(int feature_size, int shrinkage, int box_width, int box_height, int ratio1, int ratio2)
{
this->featureSize = feature_size;
@@ -37,74 +33,47 @@ struct SSDSpec
this->ratio1 = ratio1;
this->ratio2 = ratio2;
}
void print()
{
std::cout << "fsize: " << featureSize << "\tshrinkage: " << shrinkage << "\t box W:" << boxWidth << "\tbox H: " << boxHeight << "\t x ratio:" << ratio1 << "\t y ratio:" << ratio2 << std::endl;
std::cout << "fsize: " << featureSize << "\tshrinkage: " << shrinkage <<
"\t box W:" << boxWidth << "\tbox H: " << boxHeight <<
"\t x ratio:" << ratio1 << "\t y ratio:" << ratio2 << std::endl;
}
};
namespace tk
class MobilenetDetection : public DetectionNN
{
namespace dnn
{
class MobilenetDetection
{
private:
tk::dnn::NetworkRT *netRT = nullptr;
int classes;
float IoUThreshold = 0.45;
float centerVariance = 0.1;
float sizeVariance = 0.2;
float confThreshold = 0.4;
int imageSize;
float *priors = nullptr;
int nPriors = 0;
cv::Mat origImg;
float *input, *input_d;
float *locations_h, *confidences_h;
tk::dnn::dataDim_t dim;
dnnType *conf;
dnnType *loc;
float __colors[6][3] = {{1, 0, 1}, {0, 0, 1}, {0, 1, 1}, {0, 1, 0}, {1, 1, 0}, {1, 0, 0}};
int baseline = 0;
float fontScale = 0.5;
int thickness = 2;
std::vector<std::string> classesNames;
void generate_ssd_priors(const SSDSpec *specs, const int n_specs, bool clamp = true);
void convert_locatios_to_boxes_and_center();
float iou(const tk::dnn::box &a, const tk::dnn::box &b);
void preprocess();
std::vector<tk::dnn::box> postprocess(const int width, const int height);
float get_color2(int c, int x, int max);
cv::Scalar colors[256];
std::vector<std::string> classesNames;
public:
// keep track of inference times (ms)
std::vector<double> stats;
std::vector<tk::dnn::box> detected;
MobilenetDetection() {};
~MobilenetDetection() {};
MobilenetDetection() {}
~MobilenetDetection() {}
void init(std::string tensor_path, int input_size, int n_classes);
cv::Mat draw();
void update(cv::Mat &img);
bool init(const std::string& tensor_path, const int n_classes);
void preprocess(cv::Mat &frame);
void update(cv::Mat &frame);
void postprocess(dnnType **rt_out, const int n_out);
cv::Mat draw(cv::Mat &frame);
};
} // namespace dnn
} // namespace tk
#endif /*MOBILENETDETECTION_H*/
include/tkDNN/Yolo3Detection.h
+28 -65
View File
@@ -1,73 +1,36 @@
#ifndef YOLODETECTION_H
#define YOLODETECTION_H
#ifndef Yolo3Detection_H
#define Yolo3Detection_H
#include <opencv2/videoio.hpp>
#include "opencv2/opencv.hpp"
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#include <unistd.h>
#include <mutex>
#include "utils.h"
#include "DetectionNN.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
namespace tk { namespace dnn {
#include "tkdnn.h"
class Yolo3Detection : public DetectionNN
{
private:
int num = 0;
int nMasks = 0;
int nDets = 0;
tk::dnn::Yolo::detection *dets = nullptr;
tk::dnn::Yolo* yolo[3];
namespace tk { namespace dnn {
/**
*
* @author Francesco Gatti
*/
class Yolo3Detection {
private:
tk::dnn::NetworkRT *netRT = nullptr;
tk::dnn::Yolo* yolo[3];
dnnType *input, *input_d;
int ndets = 0;
tk::dnn::Yolo::detection *dets = nullptr;
cv::Mat imageF;
cv::Mat bgr[3];
public:
int classes = 0;
int num = 0;
int n_masks = 0;
float thresh = 0.3;
cv::Scalar colors[256];
// this is filled with results
std::vector<tk::dnn::box> detected;
// keep track of inference times (ms)
std::vector<double> stats;
Yolo3Detection() {}
virtual ~Yolo3Detection() {}
/**
* Method used for inizialize the class
*
* @return Success of the initialization
*/
bool init(std::string tensor_path);
cv::Mat draw(cv::Mat &frame);
void update(cv::Mat &frame);
tk::dnn::Yolo* getYoloLayer(int n=0) {
if(n<3)
return yolo[n];
else
return nullptr;
}
tk::dnn::Yolo* getYoloLayer(int n=0);
public:
Yolo3Detection() {};
~Yolo3Detection() {};
bool init(const std::string& tensor_path, const int n_classes=80);
void preprocess(cv::Mat &frame);
void update(cv::Mat &frame);
void postprocess(dnnType **rt_out, const int n_out);
cv::Mat draw(cv::Mat &frame);
};
}}
#endif /* YOLODETECTION_H*/
} // namespace dnn
} // namespace tk
#endif /* Yolo3Detection_H*/
include/tkDNN/utils.h
+1 -1
View File
@@ -14,7 +14,6 @@
#define dnnType float
#define OPENCV_CUDA
// Colored output
#define COL_END "\033[0m"
@@ -96,6 +95,7 @@ void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string
void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0, bool skipLoad = false);
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true);
void printDeviceVector(int size, dnnType* vec_d, bool device = true);
float getColor(const int c, const int x, const int max);
void resize(int size, dnnType **data);
void matrixTranspose(cublasHandle_t handle, dnnType* srcData, dnnType* dstData, int rows, int cols);