mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

Refactoring and modularization

Browse Source 
Signed-off-by: Micaela Verucchi <micaela.verucchi@unimore.it>
This commit is contained in:
Micaela Verucchi
2019-10-04 11:12:01 +02:00
parent bb7d382d96
commit 35787cc771
25 changed files with 1708 additions and 1560 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/Layer.h
+137 -105
View File
@@ -1,13 +1,17 @@
#ifndef LAYER_H
#define LAYER_H
#include<iostream>
#include <iostream>
#include "utils.h"
#include "Network.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
enum layerType_t {
enum layerType_t
{
LAYER_DENSE,
LAYER_CONV2D,
LAYER_ACTIVATION,
@@ -26,56 +30,73 @@ enum layerType_t {
/**
Simple layer Father class
*/
class Layer {
class Layer
{
public:
Layer(Network *net);
virtual ~Layer();
virtual layerType_t getLayerType() = 0;
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData) {
std::cout<<"No infer action for this layer\n";
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData)
{
std::cout << "No infer action for this layer\n";
return NULL;
}
dataDim_t input_dim, output_dim;
dnnType *dstData; //where results will be putted
dnnType *dstData; //where results will be putted
std::string getLayerName() {
std::string getLayerName()
{
layerType_t type = getLayerType();
switch(type) {
case LAYER_DENSE: return "Dense";
case LAYER_CONV2D: return "Conv2d";
case LAYER_ACTIVATION: return "Activation";
case LAYER_FLATTEN: return "Flatten";
case LAYER_MULADD: return "MulAdd";
case LAYER_POOLING: return "Pooling";
case LAYER_SOFTMAX: return "Softmax";
case LAYER_ROUTE: return "Route";
case LAYER_REORG: return "Reorg";
case LAYER_SHORTCUT: return "Shortcut";
case LAYER_UPSAMPLE: return "Upsample";
case LAYER_REGION: return "Region";
case LAYER_YOLO: return "Yolo";
default: return "unknown";
switch (type)
{
case LAYER_DENSE:
return "Dense";
case LAYER_CONV2D:
return "Conv2d";
case LAYER_ACTIVATION:
return "Activation";
case LAYER_FLATTEN:
return "Flatten";
case LAYER_MULADD:
return "MulAdd";
case LAYER_POOLING:
return "Pooling";
case LAYER_SOFTMAX:
return "Softmax";
case LAYER_ROUTE:
return "Route";
case LAYER_REORG:
return "Reorg";
case LAYER_SHORTCUT:
return "Shortcut";
case LAYER_UPSAMPLE:
return "Upsample";
case LAYER_REGION:
return "Region";
case LAYER_YOLO:
return "Yolo";
default:
return "unknown";
}
}
protected:
Network *net;
cudnnTensorDescriptor_t srcTensorDesc, dstTensorDesc;
};
/**
Father class of all layer that need to load trained weights
*/
class LayerWgs : public Layer {
class LayerWgs : public Layer
{
public:
LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt,
const char* fname_weights, bool batchnorm = false);
LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt,
const char *fname_weights, bool batchnorm = false);
virtual ~LayerWgs();
int inputs, outputs;
@@ -87,75 +108,76 @@ public:
//batchnorm
bool batchnorm;
dnnType *power_h;
dnnType *scales_h, *scales_d;
dnnType *mean_h, *mean_d;
dnnType *scales_h, *scales_d;
dnnType *mean_h, *mean_d;
dnnType *variance_h, *variance_d;
//fp16
__half *data16_h, *bias16_h;
__half *data16_d, *bias16_d;
__half *power16_h, *power16_d;
__half *scales16_h, *scales16_d;
__half *mean16_h, *mean16_d;
__half *power16_h, *power16_d;
__half *scales16_h, *scales16_d;
__half *mean16_h, *mean16_d;
__half *variance16_h, *variance16_d;
};
/**
Dense (full interconnection) layer
*/
class Dense : public LayerWgs {
class Dense : public LayerWgs
{
public:
Dense(Network *net, int out_ch, const char* fname_weights);
Dense(Network *net, int out_ch, const char *fname_weights);
virtual ~Dense();
virtual layerType_t getLayerType() { return LAYER_DENSE; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
};
/**
Avaible activation functions
*/
typedef enum {
ACTIVATION_ELU = 100,
ACTIVATION_LEAKY = 101
typedef enum
{
ACTIVATION_ELU = 100,
ACTIVATION_LEAKY = 101
} tkdnnActivationMode_t;
/**
Activation layer (it doesnt need weigths)
*/
class Activation : public Layer {
class Activation : public Layer
{
public:
int act_mode;
Activation(Network *net, int act_mode);
Activation(Network *net, int act_mode);
virtual ~Activation();
virtual layerType_t getLayerType() { return LAYER_ACTIVATION; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
protected:
cudnnActivationDescriptor_t activDesc;
};
/**
Convolutional 2D layer
*/
class Conv2d : public LayerWgs {
class Conv2d : public LayerWgs
{
public:
Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
int strideH, int strideW, int paddingH, int paddingW,
const char* fname_weights, bool batchnorm = false);
Conv2d(Network *net, int out_ch, int kernelH, int kernelW,
int strideH, int strideW, int paddingH, int paddingW,
const char *fname_weights, bool batchnorm = false);
virtual ~Conv2d();
virtual layerType_t getLayerType() { return LAYER_CONV2D; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
int kernelH, kernelW, strideH, strideW, paddingH, paddingW;
@@ -165,75 +187,74 @@ protected:
cudnnConvolutionFwdAlgo_t algo;
cudnnTensorDescriptor_t biasTensorDesc;
void* workSpace;
void *workSpace;
size_t ws_sizeInBytes;
};
/**
Flatten layer
is actually a matrix transposition
*/
class Flatten : public Layer {
class Flatten : public Layer
{
public:
Flatten(Network *net);
Flatten(Network *net);
virtual ~Flatten();
virtual layerType_t getLayerType() { return LAYER_FLATTEN; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
};
/**
MulAdd layer
apply a multiplication and then an addition for each data
*/
class MulAdd : public Layer {
class MulAdd : public Layer
{
public:
MulAdd(Network *net, dnnType mul, dnnType add);
MulAdd(Network *net, dnnType mul, dnnType add);
virtual ~MulAdd();
virtual layerType_t getLayerType() { return LAYER_MULADD; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
protected:
dnnType mul, add;
dnnType *add_vector;
};
/**
Avaible pooling functions (padding on tkDNN is not supported)
*/
typedef enum {
POOLING_MAX = 0,
POOLING_AVERAGE = 1, // count for average includes padded values
POOLING_AVERAGE_EXCLUDE_PADDING = 2 // count for average does not include padded values
typedef enum
{
POOLING_MAX = 0,
POOLING_AVERAGE = 1, // count for average includes padded values
POOLING_AVERAGE_EXCLUDE_PADDING = 2 // count for average does not include padded values
} tkdnnPoolingMode_t;
/**
Pooling layer
currenty supported only 2d pooing (also on 3d input)
*/
class Pooling : public Layer {
class Pooling : public Layer
{
public:
int winH, winW;
int strideH, strideW;
int paddingH, paddingW;
Pooling(Network *net, int winH, int winW,
int strideH, int strideW, tkdnnPoolingMode_t pool_mode);
Pooling(Network *net, int winH, int winW,
int strideH, int strideW, tkdnnPoolingMode_t pool_mode);
virtual ~Pooling();
virtual layerType_t getLayerType() { return LAYER_POOLING; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
protected:
cudnnPoolingDescriptor_t poolingDesc;
tkdnnPoolingMode_t pool_mode;
dnnType *tmpInputData, *tmpOutputData;
@@ -243,47 +264,49 @@ protected:
/**
Softmax layer
*/
class Softmax : public Layer {
class Softmax : public Layer
{
public:
Softmax(Network *net);
Softmax(Network *net);
virtual ~Softmax();
virtual layerType_t getLayerType() { return LAYER_SOFTMAX; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
};
/**
Route layer
Merge a list of layers
*/
class Route : public Layer {
class Route : public Layer
{
public:
Route(Network *net, Layer **layers, int layers_n);
Route(Network *net, Layer **layers, int layers_n);
virtual ~Route();
virtual layerType_t getLayerType() { return LAYER_ROUTE; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
public:
Layer **layers; //ids of layers to be merged
int layers_n; //number of layers
Layer **layers; //ids of layers to be merged
int layers_n; //number of layers
};
/**
Reorg layer
Mantain same dimension but change C*H*W distribution
*/
class Reorg : public Layer {
class Reorg : public Layer
{
public:
Reorg(Network *net, int stride);
virtual ~Reorg();
virtual layerType_t getLayerType() { return LAYER_REORG; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
int stride;
};
@@ -292,14 +315,15 @@ public:
Shortcut layer
sum with stride another layer
*/
class Shortcut : public Layer {
class Shortcut : public Layer
{
public:
Shortcut(Network *net, Layer *backLayer);
Shortcut(Network *net, Layer *backLayer);
virtual ~Shortcut();
virtual layerType_t getLayerType() { return LAYER_SHORTCUT; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
public:
Layer *backLayer;
@@ -309,25 +333,28 @@ public:
Upsample layer
Mantain same dimension but change C*H*W distribution
*/
class Upsample : public Layer {
class Upsample : public Layer
{
public:
Upsample(Network *net, int stride);
virtual ~Upsample();
virtual layerType_t getLayerType() { return LAYER_UPSAMPLE; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
int stride;
bool reverse;
};
struct box {
struct box
{
int cl;
float x, y, w, h;
float prob;
};
struct sortable_bbox {
struct sortable_bbox
{
int index;
int cl;
float **probs;
@@ -336,14 +363,17 @@ struct sortable_bbox {
/**
Yolo3 layer
*/
class Yolo : public Layer {
class Yolo : public Layer
{
public:
struct box {
struct box
{
float x, y, w, h;
};
struct detection{
struct detection
{
Yolo::box bbox;
int classes;
float *prob;
@@ -352,7 +382,7 @@ public:
int sort_class;
};
Yolo(Network *net, int classes, int num, const char* fname_weights);
Yolo(Network *net, int classes, int num, const char *fname_weights);
virtual ~Yolo();
virtual layerType_t getLayerType() { return LAYER_YOLO; };
@@ -360,20 +390,21 @@ public:
dnnType *mask_h, *mask_d; //anchors
dnnType *bias_h, *bias_d; //anchors
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh);
dnnType *predictions;
static const int MAX_DETECTIONS = 256;
static Yolo::detection *allocateDetections(int nboxes, int classes);
static void mergeDetections(Yolo::detection *dets, int ndets, int classes);
static void mergeDetections(Yolo::detection *dets, int ndets, int classes);
};
/**
Region layer
*/
class Region : public Layer {
class Region : public Layer
{
public:
Region(Network *net, int classes, int coords, int num);
@@ -381,15 +412,16 @@ public:
virtual layerType_t getLayerType() { return LAYER_REGION; };
int classes, coords, num;
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
};
class RegionInterpret {
class RegionInterpret
{
public:
RegionInterpret(dataDim_t input_dim, dataDim_t output_dim,
int classes, int coords, int num, float thresh, const char* fname_weights);
RegionInterpret(dataDim_t input_dim, dataDim_t output_dim,
int classes, int coords, int num, float thresh, const char *fname_weights);
~RegionInterpret();
dataDim_t input_dim, output_dim;
@@ -397,7 +429,6 @@ public:
int classes, coords, num;
float thresh;
box *boxes;
float **probs;
sortable_bbox *s;
@@ -405,9 +436,9 @@ public:
int res_boxes_n;
box get_region_box(float *x, float *biases, int n, int index, int i, int j, int w, int h, int stride);
void get_region_boxes( float *input, int w, int h, int netw, int neth, float thresh,
float **probs, box *boxes, int only_objectness,
int *map, float tree_thresh, int relative);
void get_region_boxes(float *input, int w, int h, int netw, int neth, float thresh,
float **probs, box *boxes, int only_objectness,
int *map, float tree_thresh, int relative);
void correct_region_boxes(box *boxes, int n, int w, int h, int netw, int neth, int relative);
void interpretData(dnnType *data_h, int imageW = 0, int imageH = 0);
void showImageResult(dnnType *input_h);
@@ -415,5 +446,6 @@ public:
static float box_iou(box a, box b);
};
}}
} // namespace dnn
} // namespace tk
#endif //LAYER_H
include/Network.h
+21 -14
View File
@@ -3,7 +3,10 @@
#include "utils.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
/**
Data rapresentation beetween layers
@@ -13,28 +16,31 @@ namespace tk { namespace dnn {
w = width (rows)
l = lenght (3rd dimension)
*/
struct dataDim_t {
struct dataDim_t
{
int n, c, h, w, l;
dataDim_t() : n(1), c(1), h(1), w(1), l(1) {};
dataDim_t() : n(1), c(1), h(1), w(1), l(1){};
dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) :
n(_n), c(_c), h(_h), w(_w), l(_l) {};
dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) : n(_n), c(_c), h(_h), w(_w), l(_l){};
void print() {
std::cout<<"Data dim: "<<n<<" "<<c<<" "<<h<<" "<<w<<" "<<l<<"\n";
void print()
{
std::cout << "Data dim: " << n << " " << c << " " << h << " " << w << " " << l << "\n";
}
int tot() {
return n*c*h*w*l;
int tot()
{
return n * c * h * w * l;
}
};
class Layer;
const int MAX_LAYERS = 256;
class Network {
class Network
{
public:
Network(dataDim_t input_dim);
@@ -43,7 +49,7 @@ public:
/**
Do inferece for every added layer
*/
dnnType* infer(dataDim_t &dim, dnnType* data);
dnnType *infer(dataDim_t &dim, dnnType *data);
bool addLayer(Layer *l);
void print();
@@ -53,8 +59,8 @@ public:
cudnnHandle_t cudnnHandle;
cublasHandle_t cublasHandle;
Layer* layers[MAX_LAYERS]; //contains layers of the net
int num_layers; //current number of layers
Layer *layers[MAX_LAYERS]; //contains layers of the net
int num_layers; //current number of layers
dataDim_t input_dim;
dataDim_t getOutputDim();
@@ -62,5 +68,6 @@ public:
bool fp16, dla;
};
}}
} // namespace dnn
} // namespace tk
#endif //NETWORK_H
include/NetworkRT.h
+33 -28
View File
@@ -7,17 +7,22 @@
#include "Layer.h"
#include "NvInfer.h"
namespace tk { namespace dnn {
template<typename T> void writeBUF(char*& buffer, const T& val)
namespace tk
{
*reinterpret_cast<T*>(buffer) = val;
namespace dnn
{
template <typename T>
void writeBUF(char *&buffer, const T &val)
{
*reinterpret_cast<T *>(buffer) = val;
buffer += sizeof(T);
}
template<typename T> T readBUF(const char*& buffer)
template <typename T>
T readBUF(const char *&buffer)
{
T val = *reinterpret_cast<const T*>(buffer);
T val = *reinterpret_cast<const T *>(buffer);
buffer += sizeof(T);
return val;
}
@@ -38,24 +43,23 @@ public:
YoloRT *yolos[16];
int n_yolos;
virtual IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength);
virtual IPlugin *createPlugin(const char *layerName, const void *serialData, size_t serialLength);
};
class NetworkRT {
class NetworkRT
{
public:
nvinfer1::DataType dtRT;
nvinfer1::IBuilder *builderRT;
nvinfer1::IRuntime *runtimeRT;
nvinfer1::INetworkDefinition *networkRT;
nvinfer1::INetworkDefinition *networkRT;
nvinfer1::ICudaEngine *engineRT;
nvinfer1::IExecutionContext *contextRT;
const static int MAX_BUFFERS_RT = 10;
void* buffersRT[MAX_BUFFERS_RT];
void *buffersRT[MAX_BUFFERS_RT];
int buf_input_idx, buf_output_idx;
dataDim_t input_dim, output_dim;
@@ -70,25 +74,26 @@ public:
/**
Do inferece
*/
dnnType* infer(dataDim_t &dim, dnnType* data);
void enqueue();
dnnType *infer(dataDim_t &dim, dnnType *data);
void enqueue();
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Layer *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Conv2d *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Activation *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Dense *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Layer *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Conv2d *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Activation *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Dense *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Pooling *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Softmax *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Route *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Reorg *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Region *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Shortcut *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Yolo *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Upsample *l);
bool serialize(const char *filename);
bool deserialize(const char *filename);
};
}}
} // namespace dnn
} // namespace tk
#endif //NETWORKRT_H
include/Yolo3Detection.h
+36 -29
View File
@@ -1,6 +1,9 @@
#ifndef YOLO3DDETECTION_H
#define YOLO3DDETECTION_H
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#include <stdlib.h> /* srand, rand */
#include <unistd.h>
#include <mutex>
#include "utils.h"
@@ -11,49 +14,53 @@
#include "tkdnn.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
/**
*
* @author Francesco Gatti
*/
class Yolo3Detection {
class Yolo3Detection
{
private:
tk::dnn::NetworkRT *netRT = nullptr;
tk::dnn::Yolo* yolo[3];
dnnType *input, *input_d;
private:
tk::dnn::NetworkRT *netRT = nullptr;
tk::dnn::Yolo *yolo[3];
dnnType *input, *input_d;
int ndets = 0;
tk::dnn::Yolo::detection *dets = nullptr;
int ndets = 0;
tk::dnn::Yolo::detection *dets = nullptr;
cv::Mat imageF;
cv::Mat bgr[3];
cv::Mat imageF;
cv::Mat bgr[3];
public:
int classes = 0;
int num = 0;
float thresh = 0.3;
cv::Scalar colors[256];
public:
int classes = 0;
int num = 0;
float thresh = 0.3;
cv::Scalar colors[256];
// this is filled with results
std::vector<tk::dnn::box> detected;
// this is filled with results
std::vector<tk::dnn::box> detected;
Yolo3Detection() {}
Yolo3Detection() {}
virtual ~Yolo3Detection() {}
virtual ~Yolo3Detection() {}
/**
* Method used for inizialize the class
/**
* Method used to inizialize the class
*
* @return Success of the initialization
*/
bool init(std::string tensor_path);
void addBorders(cv::Mat &imageORIG, cv::Mat &imageWBorders, int &top, int &left);
void update(cv::Mat &frame);
bool init(std::string tensor_path);
void addBorders(cv::Mat &imageORIG, cv::Mat &imageWBorders, int &top, int &left);
void update(cv::Mat &frame);
};
}}
} // namespace dnn
} // namespace tk
#endif /*YOLO3DDETECTION_H*/
include/BoxDetection.h → include/boxDetection.h
View File
include/calibration.h
+32
View File
@@ -0,0 +1,32 @@
#ifndef CALIBRATION_H
#define CALIBRATION_H
#include "gdal.h"
#include <gdal_priv.h>
#include <gdal/gdal.h>
#include "gdal/gdal_priv.h"
#include "gdal/cpl_conv.h"
#include <yaml-cpp/yaml.h>
#include <opencv2/calib3d.hpp>
#include <opencv2/core.hpp>
#include <iostream>
#include <cstring>
struct ObjCoords
{
double lat_;
double long_;
int class_;
};
void readTiff(char *filename, double *adfGeoTransform);
void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff);
void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform);
void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform);
void fillMatrix(cv::Mat &H, double *matrix, bool show = false);
void read_projection_matrix(cv::Mat &H, char *path);
void convert_coords(std::vector<ObjCoords> &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform);
#endif /*CALIBRATION_H*/
include/cameraUtils.h
+45
View File
@@ -0,0 +1,45 @@
#ifndef CAMERAUTILS_H
#define CAMERAUTILS_H
#include <vector>
#include <mutex>
#include <opencv2/core/core.hpp>
#include "tracker.h"
#include "Yolo3Detection.h"
struct Camera_t
{
int CAM_IDX;
char *input;
char *pmatrix;
char *maskfile;
char *cameraCalib;
char *maskFileOrient;
bool to_show;
tk::dnn::Yolo3Detection yolo;
double adfGeoTransform[6];
};
struct Frame_t
{
char *input;
cv::Mat frame;
int frame_nbr;
// sem_vc for mainthread, videocapturethread, originalthread and disparitythread
std::mutex sem_vc;
};
struct ModFrame_t
{
std::vector<Tracker> trackers;
geodetic_converter::GeodeticConverter gc;
double adfGeoTransform[6];
cv::Mat H;
cv::Mat original_frame;
tk::dnn::Yolo3Detection yolo;
cv::Mat mask;
// sem for mainthread, detectionthread and topviewthread
std::mutex sem;
};
#endif /*CAMERAUTILS_H*/
include/classutils.h
-316
View File
@@ -1,316 +0,0 @@
#ifndef CLASSUTILS_H
#define CLASSUTILS_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <sys/socket.h> //socket
#include <arpa/inet.h> //inet_addr
#include <unistd.h> //write
#include <opencv2/calib3d.hpp>
#include <opencv2/core.hpp>
#include "gdal.h"
#include <gdal_priv.h>
#include <gdal/gdal.h>
#include "gdal/gdal_priv.h"
#include "gdal/cpl_conv.h"
#include "tracker.h"
#include <yaml-cpp/yaml.h>
#include "../masa_protocol/include/send.hpp"
#include "../masa_protocol/include/serialize.hpp"
struct ObjCoords
{
double lat_;
double long_;
int class_;
};
void readTiff(char *filename, double *adfGeoTransform)
{
GDALDataset *poDataset;
GDALAllRegister();
poDataset = (GDALDataset *)GDALOpen(filename, GA_ReadOnly);
if (poDataset != NULL)
{
poDataset->GetGeoTransform(adfGeoTransform);
}
}
void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff)
{
YAML::Node config = YAML::LoadFile(cameraCalib);
const YAML::Node &node_test1 = config["camera_matrix"];
float data_cm[9];
for (std::size_t i = 0; i < node_test1["data"].size(); i++)
data_cm[i] = node_test1["data"][i].as<float>();
cv::Mat cameraMat_ = cv::Mat(3, 3, CV_32F, data_cm);
cameraMat = cameraMat_.clone();
std::cout << cameraMat << std::endl;
const YAML::Node &node_test2 = config["distortion_coefficients"];
float data_dc[5];
for (std::size_t i = 0; i < node_test2["data"].size(); i++)
data_dc[i] = node_test2["data"][i].as<float>();
cv::Mat distCoeff_ = cv::Mat(5, 1, CV_32F, data_dc);
distCoeff = distCoeff_.clone();
std::cout << distCoeff << std::endl;
}
void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform)
{
//Returns global coordinates from pixel x, y coordinates
double xoff, a, b, yoff, d, e;
xoff = adfGeoTransform[0];
a = adfGeoTransform[1];
b = adfGeoTransform[2];
yoff = adfGeoTransform[3];
d = adfGeoTransform[4];
e = adfGeoTransform[5];
//printf("%f %f %f %f %f %f\n",xoff, a, b, yoff, d, e );
lon = a * x + b * y + xoff;
lat = d * x + e * y + yoff;
}
void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform)
{
x = int(round((lon - adfGeoTransform[0]) / adfGeoTransform[1]));
y = int(round((lat - adfGeoTransform[3]) / adfGeoTransform[5]));
}
void fillMatrix(cv::Mat &H, double *matrix, bool show = false)
{
double *vals = (double *)H.data;
for (int i = 0; i < 9; i++)
{
vals[i] = matrix[i];
}
if (show)
std::cout << H << "\n";
}
//FILE *out_file = fopen("prova_pixel.txt", "w");
void convert_coords(std::vector<ObjCoords> &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform, int frame_nbr)
{
double latitude, longitude;
std::vector<cv::Point2f> x_y, ll;
x_y.push_back(cv::Point2f(x, y));
//transform camera pixel to map pixel
cv::perspectiveTransform(x_y, ll, H);
//tranform to map pixel to map gps
pixel2coord(ll[0].x, ll[0].y, latitude, longitude, adfGeoTransform);
//printf("lat: %f, long:%f \n", latitude, longitude);
ObjCoords coord;
coord.lat_ = latitude;
coord.long_ = longitude;
coord.class_ = detected_class;
coords.push_back(coord);
/*if (detected_class == 0)
{
struct timeval tv;
gettimeofday(&tv, NULL);
unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000;
//printf(out_file, "%d %lld %d %d\n",frame_nbr, t_stamp_ms, int(ll[0].x), int(ll[0].y));
fprintf(out_file, "%d %lld %f %f\n", frame_nbr, t_stamp_ms, coord.LAT, coord.LONG);
//printf( "%d %lld %f %f\n", frame_nbr, t_stamp_ms, coord.LAT, coord.LONG);
}*/
}
void read_projection_matrix(cv::Mat &H, char *path)
{
FILE *fp;
char *line = NULL;
size_t len = 0;
ssize_t read;
// float *proj_matrix = (float *)malloc(9 * sizeof(float));
double proj_matrix[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
int i = 0;
fp = fopen(path, "r");
if (fp == NULL)
exit(EXIT_FAILURE);
while ((read = getline(&line, &len, fp)) != -1)
{
std::cout<<line<<std::endl;
std::stringstream ss(line);
while (ss >> proj_matrix[i])
i++;
}
fclose(fp);
fillMatrix(H, proj_matrix);
free(line);
// free(proj_matrix);
}
void draw_arrow(float angleRad, float vel, cv::Scalar color, cv::Point center, cv::Mat &frame)
{
int angle = angleRad * 180.0 / CV_PI;
auto length = 10 * vel;
auto direction = cv::Point(length * cos(angleRad), length * sin(angleRad)); // calculate direction
double tipLength = .2 + 0.4 * (angle % 180) / 360;
int lineType = 8;
int thickness = 2;
cv::arrowedLine(frame, center, center + direction, color, thickness, lineType, 0, tipLength); // draw arrow!
}
unsigned long long time_in_ms()
{
struct timeval tv;
gettimeofday(&tv, NULL);
unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000;
return t_stamp_ms;
}
void addRoadUserfromTracker(const std::vector<Tracker> &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat& maskOrient, double *adfGeoTransform, cv::Mat H)
{
m->t_stamp_ms = time_in_ms();
m->objects.clear();
double lat, lon, alt;
for (auto t : trackers)
{
if (t.pred_list_.size() > 0)
{
Categories cat;
switch (t.class_)
{
case 0:
cat = Categories::C_person;
break;
case 1:
cat = Categories::C_car;
break;
case 2:
cat = Categories::C_car;
break;
case 3:
cat = Categories::C_bus;
break;
case 4:
cat = Categories::C_motorbike;
break;
case 5:
cat = Categories::C_bycicle;
break;
}
//std::cout << t.pred_list_.size() << std::endl;
gc.enu2Geodetic(t.pred_list_.back().x_, t.pred_list_.back().y_, 0, &lat, &lon, &alt);
int pix_x, pix_y;
coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
// TODO: test correctness - added perspective transform call to converter pix_x and pix_y
// sometimes some values are wrong. float ok?
// std::vector<cv::Point2f> map_p, camera_p;
// std::cout<<"--- pix_x, pix_y: "<<pix_x<<", "<<pix_y<<std::endl;
// map_p.push_back(cv::Point2f(pix_x, pix_y));
// std::cout<<"map_p: "<<map_p<<std::endl;
// //transform camera pixel to map pixel
// cv::perspectiveTransform(map_p, camera_p, H.inv());
// std::cout<<"size H: "<<H.cols<<", "<<H.rows<<std::endl;
// std::cout<<"camera_p: "<<camera_p<<std::endl;
// // TODO: in some cases these lines causes seg fault!
// std::cout<<"y, x :"<<camera_p[0].y<<", "<<camera_p[0].x<<std::endl;
// std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
// // std::cout<<"vec3b: "<<(cv::Vec3b)(pix_y,pix_x);
// assert (camera_p[0].x < maskOrient.cols);
// assert (camera_p[0].y < maskOrient.rows);
// uint8_t maskOrientPixel = maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)[0];
// std::cout<<"boo: "<<maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)<<std::endl;
// uint8_t orientation;
// if(maskOrientPixel != 0)
// {
// orientation = maskOrientPixel;
// // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
// }
// else
// {
// orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
// //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
// }
// TODO: to validate -> it works for grayscale image (see demo.cpp, row: "cv::Mat maskOrient = cv::imread(camera->maskFileOrient, 0);")
// TODO: include perspective transform
// std::cout<<"y, x :"<<pix_y<<", "<<pix_x<<std::endl;
// std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
// std::cout<<"point: "<<(cv::Point)(pix_y,pix_x);
// uint8_t maskOrientPixel = maskOrient.at<uchar>(pix_y,pix_x);
// uint8_t orientation;
// if(maskOrientPixel != 0)
// {
// orientation = maskOrientPixel;
// // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
// }
// else
// {
// orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
// //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
// }
uint8_t orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
// std::cout<<"orient: "<<unsigned(orientation)<<std::endl;
//std::cout << "lat: " << lat << " lon: " << lon << std::endl;
uint8_t velocity = uint8_t(std::abs(t.pred_list_.back().vel_ * 3.6 / 2));
// std::cout<<"vel: "<<unsigned(velocity)<<std::endl;
RoadUser r{static_cast<float>(lat), static_cast<float>(lon), velocity, orientation, cat};
//std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << int(r.speed) << " " << int(r.orientation) << " " << r.category << std::endl;
m->objects.push_back(r);
}
}
m->num_objects = m->objects.size();
}
void prepare_message(Message *m, const std::vector<ObjCoords> &coords, int idx)
{
m->cam_idx = idx;
m->t_stamp_ms = time_in_ms();
m->num_objects = coords.size();
m->objects.clear();
for (unsigned int i = 0; i < coords.size(); i++)
{
Categories cat;
switch (coords[i].class_)
{
case 0:
cat = Categories::C_person;
break;
case 1:
cat = Categories::C_car;
break;
case 2:
cat = Categories::C_car;
break;
case 3:
cat = Categories::C_bus;
break;
case 4:
cat = Categories::C_motorbike;
break;
case 5:
cat = Categories::C_bycicle;
break;
}
RoadUser r{static_cast<float>(coords[i].lat_), static_cast<float>(coords[i].long_), 0, 1, cat};
std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << cat << std::endl;
m->objects.push_back(r);
}
m->lights.clear();
}
#endif /*CLASSUTILS_H*/
include/message.h
+22
View File
@@ -0,0 +1,22 @@
#ifndef MESSAGE_H
#define MESSAGE_H
#include <iostream>
#include <cstdlib>
#include <ctime>
#include <opencv2/calib3d.hpp>
#include <opencv2/core.hpp>
// #include <sys/socket.h> //socket
// #include <arpa/inet.h> //inet_addr
// #include <unistd.h> //write
#include "tracker.h"
#include "../masa_protocol/include/send.hpp"
#include "../masa_protocol/include/serialize.hpp"
unsigned long long time_in_ms();
void addRoadUserfromTracker(const std::vector<Tracker> &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat &maskOrient, double *adfGeoTransform, cv::Mat H);
#endif /*MESSAGE_H*/
include/utils.h
+62 -48
View File
@@ -31,14 +31,18 @@
#define COL_PURPLEB "\033[1;35m"
#define COL_CYANB "\033[1;36m"
// Simple Timer
#define TIMER_START timespec start, end; \
clock_gettime(CLOCK_MONOTONIC, &start);
// Simple Timer
#define TIMER_START \
timespec start, end; \
clock_gettime(CLOCK_MONOTONIC, &start);
#define TIMER_STOP_C(col) clock_gettime(CLOCK_MONOTONIC, &end); \
double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 + \
(double)(end.tv_nsec - start.tv_nsec))/1.0e6; \
std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END;
#define TIMER_STOP_C(col) \
clock_gettime(CLOCK_MONOTONIC, &end); \
double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 + \
(double)(end.tv_nsec - start.tv_nsec)) / \
1.0e6; \
std::cout << col << "Time:" << std::setw(16) << t_ns << " ms\n" \
<< COL_END;
#define TIMER_STOP TIMER_STOP_C(COL_CYANB)
@@ -47,56 +51,66 @@
* ******************************************************/
#define EXIT_WAIVED 0
#define FatalError(s) { \
std::stringstream _where, _message; \
_where << __FILE__ << ':' << __LINE__; \
_message << std::string(s) + "\n" << __FILE__ << ':' << __LINE__;\
std::cerr << _message.str() << "\nAborting...\n"; \
cudaDeviceReset(); \
exit(EXIT_FAILURE); \
}
#define FatalError(s) \
{ \
std::stringstream _where, _message; \
_where << __FILE__ << ':' << __LINE__; \
_message << std::string(s) + "\n" \
<< __FILE__ << ':' << __LINE__; \
std::cerr << _message.str() << "\nAborting...\n"; \
cudaDeviceReset(); \
exit(EXIT_FAILURE); \
}
#define checkCUDNN(status) { \
std::stringstream _error; \
if (status != CUDNN_STATUS_SUCCESS) { \
_error << "CUDNN failure: " <<cudnnGetErrorString(status); \
FatalError(_error.str()); \
} \
}
#define checkCUDNN(status) \
{ \
std::stringstream _error; \
if (status != CUDNN_STATUS_SUCCESS) \
{ \
_error << "CUDNN failure: " << cudnnGetErrorString(status); \
FatalError(_error.str()); \
} \
}
#define checkCuda(status) { \
std::stringstream _error; \
if (status != 0) { \
_error << "Cuda failure: "<<cudaGetErrorString(status); \
FatalError(_error.str()); \
} \
}
#define checkCuda(status) \
{ \
std::stringstream _error; \
if (status != 0) \
{ \
_error << "Cuda failure: " << cudaGetErrorString(status); \
FatalError(_error.str()); \
} \
}
#define checkERROR(status) { \
std::stringstream _error; \
if (status != 0) { \
_error << "Generic failure: " << status; \
FatalError(_error.str()); \
} \
}
#define checkERROR(status) \
{ \
std::stringstream _error; \
if (status != 0) \
{ \
_error << "Generic failure: " << status; \
FatalError(_error.str()); \
} \
}
#define checkNULL(ptr) { \
std::stringstream _error; \
if (ptr == nullptr) { \
_error << "Null pointer"; \
FatalError(_error.str()); \
} \
}
#define checkNULL(ptr) \
{ \
std::stringstream _error; \
if (ptr == nullptr) \
{ \
_error << "Null pointer"; \
FatalError(_error.str()); \
} \
}
void printCenteredTitle(const char *title, char fill, int dim);
bool fileExist(const char *fname);
void readBinaryFile(const char* fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0);
void readBinaryFile(const char *fname, int size, dnnType **data_h, dnnType **data_d, int seek = 0);
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true);
void printDeviceVector(int size, dnnType* vec_d, bool device = true);
void printDeviceVector(int size, dnnType *vec_d, bool device = true);
void resize(int size, dnnType **data);
void matrixTranspose(cublasHandle_t handle, dnnType* srcData, dnnType* dstData, int rows, int cols);
void matrixTranspose(cublasHandle_t handle, dnnType *srcData, dnnType *dstData, int rows, int cols);
void matrixMulAdd( cublasHandle_t handle, dnnType* srcData, dnnType* dstData,
dnnType* add_vector, int dim, dnnType mul);
void matrixMulAdd(cublasHandle_t handle, dnnType *srcData, dnnType *dstData,
dnnType *add_vector, int dim, dnnType mul);
#endif //UTILS_H
include/visualization.h
+42
View File
@@ -0,0 +1,42 @@
#ifndef VIZUALIZATION_H
#define VIZUALIZATION_H
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
//saliency
#include <opencv2/core/utility.hpp>
#include <opencv2/saliency.hpp>
#include <opencv2/highgui.hpp>
#include <chrono>
#include <iostream>
#include <cstring>
#include "tracker.h"
#include "cameraUtils.h"
#include "calibration.h"
#include "boxDetection.h"
struct Show_t
{
cv::Mat original, detection, topview, disparity;
bool update_o, update_de, update_t, update_di;
// a single mutex for each operation - the show_updates function must get all mutex
std::mutex mutex_o, mutex_de, mutex_t, mutex_di;
};
extern Show_t updates;
extern bool gRun;
extern std::string obj_class[10];
/* Thread function to show the updated images
**/
void *show_updates(void *x_void_ptr);
void *originalFrame(void *x_void_ptr);
void *detectionFrame(void *x_void_ptr);
void *topviewFrame(void *x_void_ptr);
void *disparityFrame(void *x_void_ptr);
#endif /*VIZUALIZATION_H*/