mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

Refactoring and modularization

Browse Source 
Signed-off-by: Micaela Verucchi <micaela.verucchi@unimore.it>
This commit is contained in:
Micaela Verucchi
2019-10-04 11:12:01 +02:00
parent bb7d382d96
commit 35787cc771
25 changed files with 1708 additions and 1560 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+9 -2
View File
@@ -38,12 +38,20 @@ cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
file(GLOB tkdnn_SRC "src/*.cpp")
set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} -lcudnn -lnvinfer ${OpenCV_LIBS} -lgdal)
file(GLOB class_SRC "src/class_src/*.cpp")
set(class_LIBS ${OpenCV_LIBS} -lgdal yaml-cpp python2.7)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++11")
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES} "~/repos/cereal/include" ${CMAKE_CURRENT_SOURCE_DIR}/tracker_CLASS/c++/src)
include_directories( BEFORE ${MY_SOURCE_DIR}/src /usr/include/python2.7 )
add_library(tkDNN SHARED ${tkdnn_SRC})
target_link_libraries(tkDNN ${tkdnn_LIBS})
add_library(CLASS SHARED ${class_SRC})
target_link_libraries(CLASS ${class_LIBS})
#static
#add_library(tkDNN_static STATIC ${tkdnn_SRC})
#target_link_libraries(tkDNN_static ${tkdnn_LIBS})
@@ -104,8 +112,7 @@ add_executable(yolo3_demo demo/demo/demo.cpp
tracker_CLASS/c++/src/tracker.cpp )
target_link_libraries(yolo3_demo tkDNN)
target_link_libraries(yolo3_demo python2.7 yaml-cpp)
target_link_libraries(yolo3_demo tkDNN CLASS)
demo/demo/demo.cpp
+142 -579
View File
File diff suppressed because it is too large Load Diff
include/Layer.h
+137 -105
View File
@@ -1,13 +1,17 @@
#ifndef LAYER_H
#define LAYER_H
#include<iostream>
#include <iostream>
#include "utils.h"
#include "Network.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
enum layerType_t {
enum layerType_t
{
LAYER_DENSE,
LAYER_CONV2D,
LAYER_ACTIVATION,
@@ -26,56 +30,73 @@ enum layerType_t {
/**
Simple layer Father class
*/
class Layer {
class Layer
{
public:
Layer(Network *net);
virtual ~Layer();
virtual layerType_t getLayerType() = 0;
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData) {
std::cout<<"No infer action for this layer\n";
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData)
{
std::cout << "No infer action for this layer\n";
return NULL;
}
dataDim_t input_dim, output_dim;
dnnType *dstData; //where results will be putted
dnnType *dstData; //where results will be putted
std::string getLayerName() {
std::string getLayerName()
{
layerType_t type = getLayerType();
switch(type) {
case LAYER_DENSE: return "Dense";
case LAYER_CONV2D: return "Conv2d";
case LAYER_ACTIVATION: return "Activation";
case LAYER_FLATTEN: return "Flatten";
case LAYER_MULADD: return "MulAdd";
case LAYER_POOLING: return "Pooling";
case LAYER_SOFTMAX: return "Softmax";
case LAYER_ROUTE: return "Route";
case LAYER_REORG: return "Reorg";
case LAYER_SHORTCUT: return "Shortcut";
case LAYER_UPSAMPLE: return "Upsample";
case LAYER_REGION: return "Region";
case LAYER_YOLO: return "Yolo";
default: return "unknown";
switch (type)
{
case LAYER_DENSE:
return "Dense";
case LAYER_CONV2D:
return "Conv2d";
case LAYER_ACTIVATION:
return "Activation";
case LAYER_FLATTEN:
return "Flatten";
case LAYER_MULADD:
return "MulAdd";
case LAYER_POOLING:
return "Pooling";
case LAYER_SOFTMAX:
return "Softmax";
case LAYER_ROUTE:
return "Route";
case LAYER_REORG:
return "Reorg";
case LAYER_SHORTCUT:
return "Shortcut";
case LAYER_UPSAMPLE:
return "Upsample";
case LAYER_REGION:
return "Region";
case LAYER_YOLO:
return "Yolo";
default:
return "unknown";
}
}
protected:
Network *net;
cudnnTensorDescriptor_t srcTensorDesc, dstTensorDesc;
};
/**
Father class of all layer that need to load trained weights
*/
class LayerWgs : public Layer {
class LayerWgs : public Layer
{
public:
LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt,
const char* fname_weights, bool batchnorm = false);
LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt,
const char *fname_weights, bool batchnorm = false);
virtual ~LayerWgs();
int inputs, outputs;
@@ -87,75 +108,76 @@ public:
//batchnorm
bool batchnorm;
dnnType *power_h;
dnnType *scales_h, *scales_d;
dnnType *mean_h, *mean_d;
dnnType *scales_h, *scales_d;
dnnType *mean_h, *mean_d;
dnnType *variance_h, *variance_d;
//fp16
__half *data16_h, *bias16_h;
__half *data16_d, *bias16_d;
__half *power16_h, *power16_d;
__half *scales16_h, *scales16_d;
__half *mean16_h, *mean16_d;
__half *power16_h, *power16_d;
__half *scales16_h, *scales16_d;
__half *mean16_h, *mean16_d;
__half *variance16_h, *variance16_d;
};
/**
Dense (full interconnection) layer
*/
class Dense : public LayerWgs {
class Dense : public LayerWgs
{
public:
Dense(Network *net, int out_ch, const char* fname_weights);
Dense(Network *net, int out_ch, const char *fname_weights);
virtual ~Dense();
virtual layerType_t getLayerType() { return LAYER_DENSE; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
};
/**
Avaible activation functions
*/
typedef enum {
ACTIVATION_ELU = 100,
ACTIVATION_LEAKY = 101
typedef enum
{
ACTIVATION_ELU = 100,
ACTIVATION_LEAKY = 101
} tkdnnActivationMode_t;
/**
Activation layer (it doesnt need weigths)
*/
class Activation : public Layer {
class Activation : public Layer
{
public:
int act_mode;
Activation(Network *net, int act_mode);
Activation(Network *net, int act_mode);
virtual ~Activation();
virtual layerType_t getLayerType() { return LAYER_ACTIVATION; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
protected:
cudnnActivationDescriptor_t activDesc;
};
/**
Convolutional 2D layer
*/
class Conv2d : public LayerWgs {
class Conv2d : public LayerWgs
{
public:
Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
int strideH, int strideW, int paddingH, int paddingW,
const char* fname_weights, bool batchnorm = false);
Conv2d(Network *net, int out_ch, int kernelH, int kernelW,
int strideH, int strideW, int paddingH, int paddingW,
const char *fname_weights, bool batchnorm = false);
virtual ~Conv2d();
virtual layerType_t getLayerType() { return LAYER_CONV2D; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
int kernelH, kernelW, strideH, strideW, paddingH, paddingW;
@@ -165,75 +187,74 @@ protected:
cudnnConvolutionFwdAlgo_t algo;
cudnnTensorDescriptor_t biasTensorDesc;
void* workSpace;
void *workSpace;
size_t ws_sizeInBytes;
};
/**
Flatten layer
is actually a matrix transposition
*/
class Flatten : public Layer {
class Flatten : public Layer
{
public:
Flatten(Network *net);
Flatten(Network *net);
virtual ~Flatten();
virtual layerType_t getLayerType() { return LAYER_FLATTEN; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
};
/**
MulAdd layer
apply a multiplication and then an addition for each data
*/
class MulAdd : public Layer {
class MulAdd : public Layer
{
public:
MulAdd(Network *net, dnnType mul, dnnType add);
MulAdd(Network *net, dnnType mul, dnnType add);
virtual ~MulAdd();
virtual layerType_t getLayerType() { return LAYER_MULADD; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
protected:
dnnType mul, add;
dnnType *add_vector;
};
/**
Avaible pooling functions (padding on tkDNN is not supported)
*/
typedef enum {
POOLING_MAX = 0,
POOLING_AVERAGE = 1, // count for average includes padded values
POOLING_AVERAGE_EXCLUDE_PADDING = 2 // count for average does not include padded values
typedef enum
{
POOLING_MAX = 0,
POOLING_AVERAGE = 1, // count for average includes padded values
POOLING_AVERAGE_EXCLUDE_PADDING = 2 // count for average does not include padded values
} tkdnnPoolingMode_t;
/**
Pooling layer
currenty supported only 2d pooing (also on 3d input)
*/
class Pooling : public Layer {
class Pooling : public Layer
{
public:
int winH, winW;
int strideH, strideW;
int paddingH, paddingW;
Pooling(Network *net, int winH, int winW,
int strideH, int strideW, tkdnnPoolingMode_t pool_mode);
Pooling(Network *net, int winH, int winW,
int strideH, int strideW, tkdnnPoolingMode_t pool_mode);
virtual ~Pooling();
virtual layerType_t getLayerType() { return LAYER_POOLING; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
protected:
cudnnPoolingDescriptor_t poolingDesc;
tkdnnPoolingMode_t pool_mode;
dnnType *tmpInputData, *tmpOutputData;
@@ -243,47 +264,49 @@ protected:
/**
Softmax layer
*/
class Softmax : public Layer {
class Softmax : public Layer
{
public:
Softmax(Network *net);
Softmax(Network *net);
virtual ~Softmax();
virtual layerType_t getLayerType() { return LAYER_SOFTMAX; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
};
/**
Route layer
Merge a list of layers
*/
class Route : public Layer {
class Route : public Layer
{
public:
Route(Network *net, Layer **layers, int layers_n);
Route(Network *net, Layer **layers, int layers_n);
virtual ~Route();
virtual layerType_t getLayerType() { return LAYER_ROUTE; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
public:
Layer **layers; //ids of layers to be merged
int layers_n; //number of layers
Layer **layers; //ids of layers to be merged
int layers_n; //number of layers
};
/**
Reorg layer
Mantain same dimension but change C*H*W distribution
*/
class Reorg : public Layer {
class Reorg : public Layer
{
public:
Reorg(Network *net, int stride);
virtual ~Reorg();
virtual layerType_t getLayerType() { return LAYER_REORG; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
int stride;
};
@@ -292,14 +315,15 @@ public:
Shortcut layer
sum with stride another layer
*/
class Shortcut : public Layer {
class Shortcut : public Layer
{
public:
Shortcut(Network *net, Layer *backLayer);
Shortcut(Network *net, Layer *backLayer);
virtual ~Shortcut();
virtual layerType_t getLayerType() { return LAYER_SHORTCUT; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
public:
Layer *backLayer;
@@ -309,25 +333,28 @@ public:
Upsample layer
Mantain same dimension but change C*H*W distribution
*/
class Upsample : public Layer {
class Upsample : public Layer
{
public:
Upsample(Network *net, int stride);
virtual ~Upsample();
virtual layerType_t getLayerType() { return LAYER_UPSAMPLE; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
int stride;
bool reverse;
};
struct box {
struct box
{
int cl;
float x, y, w, h;
float prob;
};
struct sortable_bbox {
struct sortable_bbox
{
int index;
int cl;
float **probs;
@@ -336,14 +363,17 @@ struct sortable_bbox {
/**
Yolo3 layer
*/
class Yolo : public Layer {
class Yolo : public Layer
{
public:
struct box {
struct box
{
float x, y, w, h;
};
struct detection{
struct detection
{
Yolo::box bbox;
int classes;
float *prob;
@@ -352,7 +382,7 @@ public:
int sort_class;
};
Yolo(Network *net, int classes, int num, const char* fname_weights);
Yolo(Network *net, int classes, int num, const char *fname_weights);
virtual ~Yolo();
virtual layerType_t getLayerType() { return LAYER_YOLO; };
@@ -360,20 +390,21 @@ public:
dnnType *mask_h, *mask_d; //anchors
dnnType *bias_h, *bias_d; //anchors
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh);
dnnType *predictions;
static const int MAX_DETECTIONS = 256;
static Yolo::detection *allocateDetections(int nboxes, int classes);
static void mergeDetections(Yolo::detection *dets, int ndets, int classes);
static void mergeDetections(Yolo::detection *dets, int ndets, int classes);
};
/**
Region layer
*/
class Region : public Layer {
class Region : public Layer
{
public:
Region(Network *net, int classes, int coords, int num);
@@ -381,15 +412,16 @@ public:
virtual layerType_t getLayerType() { return LAYER_REGION; };
int classes, coords, num;
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
};
class RegionInterpret {
class RegionInterpret
{
public:
RegionInterpret(dataDim_t input_dim, dataDim_t output_dim,
int classes, int coords, int num, float thresh, const char* fname_weights);
RegionInterpret(dataDim_t input_dim, dataDim_t output_dim,
int classes, int coords, int num, float thresh, const char *fname_weights);
~RegionInterpret();
dataDim_t input_dim, output_dim;
@@ -397,7 +429,6 @@ public:
int classes, coords, num;
float thresh;
box *boxes;
float **probs;
sortable_bbox *s;
@@ -405,9 +436,9 @@ public:
int res_boxes_n;
box get_region_box(float *x, float *biases, int n, int index, int i, int j, int w, int h, int stride);
void get_region_boxes( float *input, int w, int h, int netw, int neth, float thresh,
float **probs, box *boxes, int only_objectness,
int *map, float tree_thresh, int relative);
void get_region_boxes(float *input, int w, int h, int netw, int neth, float thresh,
float **probs, box *boxes, int only_objectness,
int *map, float tree_thresh, int relative);
void correct_region_boxes(box *boxes, int n, int w, int h, int netw, int neth, int relative);
void interpretData(dnnType *data_h, int imageW = 0, int imageH = 0);
void showImageResult(dnnType *input_h);
@@ -415,5 +446,6 @@ public:
static float box_iou(box a, box b);
};
}}
} // namespace dnn
} // namespace tk
#endif //LAYER_H
include/Network.h
+21 -14
View File
@@ -3,7 +3,10 @@
#include "utils.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
/**
Data rapresentation beetween layers
@@ -13,28 +16,31 @@ namespace tk { namespace dnn {
w = width (rows)
l = lenght (3rd dimension)
*/
struct dataDim_t {
struct dataDim_t
{
int n, c, h, w, l;
dataDim_t() : n(1), c(1), h(1), w(1), l(1) {};
dataDim_t() : n(1), c(1), h(1), w(1), l(1){};
dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) :
n(_n), c(_c), h(_h), w(_w), l(_l) {};
dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) : n(_n), c(_c), h(_h), w(_w), l(_l){};
void print() {
std::cout<<"Data dim: "<<n<<" "<<c<<" "<<h<<" "<<w<<" "<<l<<"\n";
void print()
{
std::cout << "Data dim: " << n << " " << c << " " << h << " " << w << " " << l << "\n";
}
int tot() {
return n*c*h*w*l;
int tot()
{
return n * c * h * w * l;
}
};
class Layer;
const int MAX_LAYERS = 256;
class Network {
class Network
{
public:
Network(dataDim_t input_dim);
@@ -43,7 +49,7 @@ public:
/**
Do inferece for every added layer
*/
dnnType* infer(dataDim_t &dim, dnnType* data);
dnnType *infer(dataDim_t &dim, dnnType *data);
bool addLayer(Layer *l);
void print();
@@ -53,8 +59,8 @@ public:
cudnnHandle_t cudnnHandle;
cublasHandle_t cublasHandle;
Layer* layers[MAX_LAYERS]; //contains layers of the net
int num_layers; //current number of layers
Layer *layers[MAX_LAYERS]; //contains layers of the net
int num_layers; //current number of layers
dataDim_t input_dim;
dataDim_t getOutputDim();
@@ -62,5 +68,6 @@ public:
bool fp16, dla;
};
}}
} // namespace dnn
} // namespace tk
#endif //NETWORK_H
include/NetworkRT.h
+33 -28
View File
@@ -7,17 +7,22 @@
#include "Layer.h"
#include "NvInfer.h"
namespace tk { namespace dnn {
template<typename T> void writeBUF(char*& buffer, const T& val)
namespace tk
{
*reinterpret_cast<T*>(buffer) = val;
namespace dnn
{
template <typename T>
void writeBUF(char *&buffer, const T &val)
{
*reinterpret_cast<T *>(buffer) = val;
buffer += sizeof(T);
}
template<typename T> T readBUF(const char*& buffer)
template <typename T>
T readBUF(const char *&buffer)
{
T val = *reinterpret_cast<const T*>(buffer);
T val = *reinterpret_cast<const T *>(buffer);
buffer += sizeof(T);
return val;
}
@@ -38,24 +43,23 @@ public:
YoloRT *yolos[16];
int n_yolos;
virtual IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength);
virtual IPlugin *createPlugin(const char *layerName, const void *serialData, size_t serialLength);
};
class NetworkRT {
class NetworkRT
{
public:
nvinfer1::DataType dtRT;
nvinfer1::IBuilder *builderRT;
nvinfer1::IRuntime *runtimeRT;
nvinfer1::INetworkDefinition *networkRT;
nvinfer1::INetworkDefinition *networkRT;
nvinfer1::ICudaEngine *engineRT;
nvinfer1::IExecutionContext *contextRT;
const static int MAX_BUFFERS_RT = 10;
void* buffersRT[MAX_BUFFERS_RT];
void *buffersRT[MAX_BUFFERS_RT];
int buf_input_idx, buf_output_idx;
dataDim_t input_dim, output_dim;
@@ -70,25 +74,26 @@ public:
/**
Do inferece
*/
dnnType* infer(dataDim_t &dim, dnnType* data);
void enqueue();
dnnType *infer(dataDim_t &dim, dnnType *data);
void enqueue();
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Layer *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Conv2d *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Activation *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Dense *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Layer *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Conv2d *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Activation *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Dense *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Pooling *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Softmax *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Route *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Reorg *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Region *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Shortcut *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Yolo *l);
nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Upsample *l);
bool serialize(const char *filename);
bool deserialize(const char *filename);
};
}}
} // namespace dnn
} // namespace tk
#endif //NETWORKRT_H
include/Yolo3Detection.h
+36 -29
View File
@@ -1,6 +1,9 @@
#ifndef YOLO3DDETECTION_H
#define YOLO3DDETECTION_H
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#include <stdlib.h> /* srand, rand */
#include <unistd.h>
#include <mutex>
#include "utils.h"
@@ -11,49 +14,53 @@
#include "tkdnn.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
/**
*
* @author Francesco Gatti
*/
class Yolo3Detection {
class Yolo3Detection
{
private:
tk::dnn::NetworkRT *netRT = nullptr;
tk::dnn::Yolo* yolo[3];
dnnType *input, *input_d;
private:
tk::dnn::NetworkRT *netRT = nullptr;
tk::dnn::Yolo *yolo[3];
dnnType *input, *input_d;
int ndets = 0;
tk::dnn::Yolo::detection *dets = nullptr;
int ndets = 0;
tk::dnn::Yolo::detection *dets = nullptr;
cv::Mat imageF;
cv::Mat bgr[3];
cv::Mat imageF;
cv::Mat bgr[3];
public:
int classes = 0;
int num = 0;
float thresh = 0.3;
cv::Scalar colors[256];
public:
int classes = 0;
int num = 0;
float thresh = 0.3;
cv::Scalar colors[256];
// this is filled with results
std::vector<tk::dnn::box> detected;
// this is filled with results
std::vector<tk::dnn::box> detected;
Yolo3Detection() {}
Yolo3Detection() {}
virtual ~Yolo3Detection() {}
virtual ~Yolo3Detection() {}
/**
* Method used for inizialize the class
/**
* Method used to inizialize the class
*
* @return Success of the initialization
*/
bool init(std::string tensor_path);
void addBorders(cv::Mat &imageORIG, cv::Mat &imageWBorders, int &top, int &left);
void update(cv::Mat &frame);
bool init(std::string tensor_path);
void addBorders(cv::Mat &imageORIG, cv::Mat &imageWBorders, int &top, int &left);
void update(cv::Mat &frame);
};
}}
} // namespace dnn
} // namespace tk
#endif /*YOLO3DDETECTION_H*/
include/BoxDetection.h → include/boxDetection.h
View File
include/calibration.h
+32
View File
@@ -0,0 +1,32 @@
#ifndef CALIBRATION_H
#define CALIBRATION_H
#include "gdal.h"
#include <gdal_priv.h>
#include <gdal/gdal.h>
#include "gdal/gdal_priv.h"
#include "gdal/cpl_conv.h"
#include <yaml-cpp/yaml.h>
#include <opencv2/calib3d.hpp>
#include <opencv2/core.hpp>
#include <iostream>
#include <cstring>
struct ObjCoords
{
double lat_;
double long_;
int class_;
};
void readTiff(char *filename, double *adfGeoTransform);
void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff);
void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform);
void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform);
void fillMatrix(cv::Mat &H, double *matrix, bool show = false);
void read_projection_matrix(cv::Mat &H, char *path);
void convert_coords(std::vector<ObjCoords> &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform);
#endif /*CALIBRATION_H*/
include/cameraUtils.h
+45
View File
@@ -0,0 +1,45 @@
#ifndef CAMERAUTILS_H
#define CAMERAUTILS_H
#include <vector>
#include <mutex>
#include <opencv2/core/core.hpp>
#include "tracker.h"
#include "Yolo3Detection.h"
struct Camera_t
{
int CAM_IDX;
char *input;
char *pmatrix;
char *maskfile;
char *cameraCalib;
char *maskFileOrient;
bool to_show;
tk::dnn::Yolo3Detection yolo;
double adfGeoTransform[6];
};
struct Frame_t
{
char *input;
cv::Mat frame;
int frame_nbr;
// sem_vc for mainthread, videocapturethread, originalthread and disparitythread
std::mutex sem_vc;
};
struct ModFrame_t
{
std::vector<Tracker> trackers;
geodetic_converter::GeodeticConverter gc;
double adfGeoTransform[6];
cv::Mat H;
cv::Mat original_frame;
tk::dnn::Yolo3Detection yolo;
cv::Mat mask;
// sem for mainthread, detectionthread and topviewthread
std::mutex sem;
};
#endif /*CAMERAUTILS_H*/
include/classutils.h
-316
View File
@@ -1,316 +0,0 @@
#ifndef CLASSUTILS_H
#define CLASSUTILS_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <sys/socket.h> //socket
#include <arpa/inet.h> //inet_addr
#include <unistd.h> //write
#include <opencv2/calib3d.hpp>
#include <opencv2/core.hpp>
#include "gdal.h"
#include <gdal_priv.h>
#include <gdal/gdal.h>
#include "gdal/gdal_priv.h"
#include "gdal/cpl_conv.h"
#include "tracker.h"
#include <yaml-cpp/yaml.h>
#include "../masa_protocol/include/send.hpp"
#include "../masa_protocol/include/serialize.hpp"
struct ObjCoords
{
double lat_;
double long_;
int class_;
};
void readTiff(char *filename, double *adfGeoTransform)
{
GDALDataset *poDataset;
GDALAllRegister();
poDataset = (GDALDataset *)GDALOpen(filename, GA_ReadOnly);
if (poDataset != NULL)
{
poDataset->GetGeoTransform(adfGeoTransform);
}
}
void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff)
{
YAML::Node config = YAML::LoadFile(cameraCalib);
const YAML::Node &node_test1 = config["camera_matrix"];
float data_cm[9];
for (std::size_t i = 0; i < node_test1["data"].size(); i++)
data_cm[i] = node_test1["data"][i].as<float>();
cv::Mat cameraMat_ = cv::Mat(3, 3, CV_32F, data_cm);
cameraMat = cameraMat_.clone();
std::cout << cameraMat << std::endl;
const YAML::Node &node_test2 = config["distortion_coefficients"];
float data_dc[5];
for (std::size_t i = 0; i < node_test2["data"].size(); i++)
data_dc[i] = node_test2["data"][i].as<float>();
cv::Mat distCoeff_ = cv::Mat(5, 1, CV_32F, data_dc);
distCoeff = distCoeff_.clone();
std::cout << distCoeff << std::endl;
}
void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform)
{
//Returns global coordinates from pixel x, y coordinates
double xoff, a, b, yoff, d, e;
xoff = adfGeoTransform[0];
a = adfGeoTransform[1];
b = adfGeoTransform[2];
yoff = adfGeoTransform[3];
d = adfGeoTransform[4];
e = adfGeoTransform[5];
//printf("%f %f %f %f %f %f\n",xoff, a, b, yoff, d, e );
lon = a * x + b * y + xoff;
lat = d * x + e * y + yoff;
}
void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform)
{
x = int(round((lon - adfGeoTransform[0]) / adfGeoTransform[1]));
y = int(round((lat - adfGeoTransform[3]) / adfGeoTransform[5]));
}
void fillMatrix(cv::Mat &H, double *matrix, bool show = false)
{
double *vals = (double *)H.data;
for (int i = 0; i < 9; i++)
{
vals[i] = matrix[i];
}
if (show)
std::cout << H << "\n";
}
//FILE *out_file = fopen("prova_pixel.txt", "w");
void convert_coords(std::vector<ObjCoords> &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform, int frame_nbr)
{
double latitude, longitude;
std::vector<cv::Point2f> x_y, ll;
x_y.push_back(cv::Point2f(x, y));
//transform camera pixel to map pixel
cv::perspectiveTransform(x_y, ll, H);
//tranform to map pixel to map gps
pixel2coord(ll[0].x, ll[0].y, latitude, longitude, adfGeoTransform);
//printf("lat: %f, long:%f \n", latitude, longitude);
ObjCoords coord;
coord.lat_ = latitude;
coord.long_ = longitude;
coord.class_ = detected_class;
coords.push_back(coord);
/*if (detected_class == 0)
{
struct timeval tv;
gettimeofday(&tv, NULL);
unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000;
//printf(out_file, "%d %lld %d %d\n",frame_nbr, t_stamp_ms, int(ll[0].x), int(ll[0].y));
fprintf(out_file, "%d %lld %f %f\n", frame_nbr, t_stamp_ms, coord.LAT, coord.LONG);
//printf( "%d %lld %f %f\n", frame_nbr, t_stamp_ms, coord.LAT, coord.LONG);
}*/
}
void read_projection_matrix(cv::Mat &H, char *path)
{
FILE *fp;
char *line = NULL;
size_t len = 0;
ssize_t read;
// float *proj_matrix = (float *)malloc(9 * sizeof(float));
double proj_matrix[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
int i = 0;
fp = fopen(path, "r");
if (fp == NULL)
exit(EXIT_FAILURE);
while ((read = getline(&line, &len, fp)) != -1)
{
std::cout<<line<<std::endl;
std::stringstream ss(line);
while (ss >> proj_matrix[i])
i++;
}
fclose(fp);
fillMatrix(H, proj_matrix);
free(line);
// free(proj_matrix);
}
void draw_arrow(float angleRad, float vel, cv::Scalar color, cv::Point center, cv::Mat &frame)
{
int angle = angleRad * 180.0 / CV_PI;
auto length = 10 * vel;
auto direction = cv::Point(length * cos(angleRad), length * sin(angleRad)); // calculate direction
double tipLength = .2 + 0.4 * (angle % 180) / 360;
int lineType = 8;
int thickness = 2;
cv::arrowedLine(frame, center, center + direction, color, thickness, lineType, 0, tipLength); // draw arrow!
}
unsigned long long time_in_ms()
{
struct timeval tv;
gettimeofday(&tv, NULL);
unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000;
return t_stamp_ms;
}
void addRoadUserfromTracker(const std::vector<Tracker> &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat& maskOrient, double *adfGeoTransform, cv::Mat H)
{
m->t_stamp_ms = time_in_ms();
m->objects.clear();
double lat, lon, alt;
for (auto t : trackers)
{
if (t.pred_list_.size() > 0)
{
Categories cat;
switch (t.class_)
{
case 0:
cat = Categories::C_person;
break;
case 1:
cat = Categories::C_car;
break;
case 2:
cat = Categories::C_car;
break;
case 3:
cat = Categories::C_bus;
break;
case 4:
cat = Categories::C_motorbike;
break;
case 5:
cat = Categories::C_bycicle;
break;
}
//std::cout << t.pred_list_.size() << std::endl;
gc.enu2Geodetic(t.pred_list_.back().x_, t.pred_list_.back().y_, 0, &lat, &lon, &alt);
int pix_x, pix_y;
coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
// TODO: test correctness - added perspective transform call to converter pix_x and pix_y
// sometimes some values are wrong. float ok?
// std::vector<cv::Point2f> map_p, camera_p;
// std::cout<<"--- pix_x, pix_y: "<<pix_x<<", "<<pix_y<<std::endl;
// map_p.push_back(cv::Point2f(pix_x, pix_y));
// std::cout<<"map_p: "<<map_p<<std::endl;
// //transform camera pixel to map pixel
// cv::perspectiveTransform(map_p, camera_p, H.inv());
// std::cout<<"size H: "<<H.cols<<", "<<H.rows<<std::endl;
// std::cout<<"camera_p: "<<camera_p<<std::endl;
// // TODO: in some cases these lines causes seg fault!
// std::cout<<"y, x :"<<camera_p[0].y<<", "<<camera_p[0].x<<std::endl;
// std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
// // std::cout<<"vec3b: "<<(cv::Vec3b)(pix_y,pix_x);
// assert (camera_p[0].x < maskOrient.cols);
// assert (camera_p[0].y < maskOrient.rows);
// uint8_t maskOrientPixel = maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)[0];
// std::cout<<"boo: "<<maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)<<std::endl;
// uint8_t orientation;
// if(maskOrientPixel != 0)
// {
// orientation = maskOrientPixel;
// // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
// }
// else
// {
// orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
// //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
// }
// TODO: to validate -> it works for grayscale image (see demo.cpp, row: "cv::Mat maskOrient = cv::imread(camera->maskFileOrient, 0);")
// TODO: include perspective transform
// std::cout<<"y, x :"<<pix_y<<", "<<pix_x<<std::endl;
// std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
// std::cout<<"point: "<<(cv::Point)(pix_y,pix_x);
// uint8_t maskOrientPixel = maskOrient.at<uchar>(pix_y,pix_x);
// uint8_t orientation;
// if(maskOrientPixel != 0)
// {
// orientation = maskOrientPixel;
// // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
// }
// else
// {
// orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
// //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
// }
uint8_t orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
// std::cout<<"orient: "<<unsigned(orientation)<<std::endl;
//std::cout << "lat: " << lat << " lon: " << lon << std::endl;
uint8_t velocity = uint8_t(std::abs(t.pred_list_.back().vel_ * 3.6 / 2));
// std::cout<<"vel: "<<unsigned(velocity)<<std::endl;
RoadUser r{static_cast<float>(lat), static_cast<float>(lon), velocity, orientation, cat};
//std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << int(r.speed) << " " << int(r.orientation) << " " << r.category << std::endl;
m->objects.push_back(r);
}
}
m->num_objects = m->objects.size();
}
void prepare_message(Message *m, const std::vector<ObjCoords> &coords, int idx)
{
m->cam_idx = idx;
m->t_stamp_ms = time_in_ms();
m->num_objects = coords.size();
m->objects.clear();
for (unsigned int i = 0; i < coords.size(); i++)
{
Categories cat;
switch (coords[i].class_)
{
case 0:
cat = Categories::C_person;
break;
case 1:
cat = Categories::C_car;
break;
case 2:
cat = Categories::C_car;
break;
case 3:
cat = Categories::C_bus;
break;
case 4:
cat = Categories::C_motorbike;
break;
case 5:
cat = Categories::C_bycicle;
break;
}
RoadUser r{static_cast<float>(coords[i].lat_), static_cast<float>(coords[i].long_), 0, 1, cat};
std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << cat << std::endl;
m->objects.push_back(r);
}
m->lights.clear();
}
#endif /*CLASSUTILS_H*/
include/message.h
+22
View File
@@ -0,0 +1,22 @@
#ifndef MESSAGE_H
#define MESSAGE_H
#include <iostream>
#include <cstdlib>
#include <ctime>
#include <opencv2/calib3d.hpp>
#include <opencv2/core.hpp>
// #include <sys/socket.h> //socket
// #include <arpa/inet.h> //inet_addr
// #include <unistd.h> //write
#include "tracker.h"
#include "../masa_protocol/include/send.hpp"
#include "../masa_protocol/include/serialize.hpp"
unsigned long long time_in_ms();
void addRoadUserfromTracker(const std::vector<Tracker> &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat &maskOrient, double *adfGeoTransform, cv::Mat H);
#endif /*MESSAGE_H*/
include/utils.h
+62 -48
View File
@@ -31,14 +31,18 @@
#define COL_PURPLEB "\033[1;35m"
#define COL_CYANB "\033[1;36m"
// Simple Timer
#define TIMER_START timespec start, end; \
clock_gettime(CLOCK_MONOTONIC, &start);
// Simple Timer
#define TIMER_START \
timespec start, end; \
clock_gettime(CLOCK_MONOTONIC, &start);
#define TIMER_STOP_C(col) clock_gettime(CLOCK_MONOTONIC, &end); \
double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 + \
(double)(end.tv_nsec - start.tv_nsec))/1.0e6; \
std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END;
#define TIMER_STOP_C(col) \
clock_gettime(CLOCK_MONOTONIC, &end); \
double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 + \
(double)(end.tv_nsec - start.tv_nsec)) / \
1.0e6; \
std::cout << col << "Time:" << std::setw(16) << t_ns << " ms\n" \
<< COL_END;
#define TIMER_STOP TIMER_STOP_C(COL_CYANB)
@@ -47,56 +51,66 @@
* ******************************************************/
#define EXIT_WAIVED 0
#define FatalError(s) { \
std::stringstream _where, _message; \
_where << __FILE__ << ':' << __LINE__; \
_message << std::string(s) + "\n" << __FILE__ << ':' << __LINE__;\
std::cerr << _message.str() << "\nAborting...\n"; \
cudaDeviceReset(); \
exit(EXIT_FAILURE); \
}
#define FatalError(s) \
{ \
std::stringstream _where, _message; \
_where << __FILE__ << ':' << __LINE__; \
_message << std::string(s) + "\n" \
<< __FILE__ << ':' << __LINE__; \
std::cerr << _message.str() << "\nAborting...\n"; \
cudaDeviceReset(); \
exit(EXIT_FAILURE); \
}
#define checkCUDNN(status) { \
std::stringstream _error; \
if (status != CUDNN_STATUS_SUCCESS) { \
_error << "CUDNN failure: " <<cudnnGetErrorString(status); \
FatalError(_error.str()); \
} \
}
#define checkCUDNN(status) \
{ \
std::stringstream _error; \
if (status != CUDNN_STATUS_SUCCESS) \
{ \
_error << "CUDNN failure: " << cudnnGetErrorString(status); \
FatalError(_error.str()); \
} \
}
#define checkCuda(status) { \
std::stringstream _error; \
if (status != 0) { \
_error << "Cuda failure: "<<cudaGetErrorString(status); \
FatalError(_error.str()); \
} \
}
#define checkCuda(status) \
{ \
std::stringstream _error; \
if (status != 0) \
{ \
_error << "Cuda failure: " << cudaGetErrorString(status); \
FatalError(_error.str()); \
} \
}
#define checkERROR(status) { \
std::stringstream _error; \
if (status != 0) { \
_error << "Generic failure: " << status; \
FatalError(_error.str()); \
} \
}
#define checkERROR(status) \
{ \
std::stringstream _error; \
if (status != 0) \
{ \
_error << "Generic failure: " << status; \
FatalError(_error.str()); \
} \
}
#define checkNULL(ptr) { \
std::stringstream _error; \
if (ptr == nullptr) { \
_error << "Null pointer"; \
FatalError(_error.str()); \
} \
}
#define checkNULL(ptr) \
{ \
std::stringstream _error; \
if (ptr == nullptr) \
{ \
_error << "Null pointer"; \
FatalError(_error.str()); \
} \
}
void printCenteredTitle(const char *title, char fill, int dim);
bool fileExist(const char *fname);
void readBinaryFile(const char* fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0);
void readBinaryFile(const char *fname, int size, dnnType **data_h, dnnType **data_d, int seek = 0);
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true);
void printDeviceVector(int size, dnnType* vec_d, bool device = true);
void printDeviceVector(int size, dnnType *vec_d, bool device = true);
void resize(int size, dnnType **data);
void matrixTranspose(cublasHandle_t handle, dnnType* srcData, dnnType* dstData, int rows, int cols);
void matrixTranspose(cublasHandle_t handle, dnnType *srcData, dnnType *dstData, int rows, int cols);
void matrixMulAdd( cublasHandle_t handle, dnnType* srcData, dnnType* dstData,
dnnType* add_vector, int dim, dnnType mul);
void matrixMulAdd(cublasHandle_t handle, dnnType *srcData, dnnType *dstData,
dnnType *add_vector, int dim, dnnType mul);
#endif //UTILS_H
include/visualization.h
+42
View File
@@ -0,0 +1,42 @@
#ifndef VIZUALIZATION_H
#define VIZUALIZATION_H
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
//saliency
#include <opencv2/core/utility.hpp>
#include <opencv2/saliency.hpp>
#include <opencv2/highgui.hpp>
#include <chrono>
#include <iostream>
#include <cstring>
#include "tracker.h"
#include "cameraUtils.h"
#include "calibration.h"
#include "boxDetection.h"
struct Show_t
{
cv::Mat original, detection, topview, disparity;
bool update_o, update_de, update_t, update_di;
// a single mutex for each operation - the show_updates function must get all mutex
std::mutex mutex_o, mutex_de, mutex_t, mutex_di;
};
extern Show_t updates;
extern bool gRun;
extern std::string obj_class[10];
/* Thread function to show the updated images
**/
void *show_updates(void *x_void_ptr);
void *originalFrame(void *x_void_ptr);
void *detectionFrame(void *x_void_ptr);
void *topviewFrame(void *x_void_ptr);
void *disparityFrame(void *x_void_ptr);
#endif /*VIZUALIZATION_H*/
src/Activation.cpp
+49 -41
View File
@@ -3,64 +3,72 @@
#include "Layer.h"
#include "kernels.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
Activation::Activation(Network *net, int act_mode) :
Layer(net) {
Activation::Activation(Network *net, int act_mode) : Layer(net)
{
this->act_mode = act_mode;
checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
checkCuda(cudaMalloc(&dstData, input_dim.tot() * sizeof(dnnType)));
if(int(act_mode) < 100) {
if (int(act_mode) < 100)
{
checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc,
net->tensorFormat,
net->dataType,
input_dim.n*input_dim.l,
input_dim.c,
input_dim.h, input_dim.w) );
checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
net->tensorFormat,
net->dataType,
input_dim.n*input_dim.l,
input_dim.c,
input_dim.h, input_dim.w) );
checkCUDNN(cudnnSetTensor4dDescriptor(srcTensorDesc,
net->tensorFormat,
net->dataType,
input_dim.n * input_dim.l,
input_dim.c,
input_dim.h, input_dim.w));
checkCUDNN(cudnnSetTensor4dDescriptor(dstTensorDesc,
net->tensorFormat,
net->dataType,
input_dim.n * input_dim.l,
input_dim.c,
input_dim.h, input_dim.w));
checkCUDNN( cudnnCreateActivationDescriptor(&activDesc) );
checkCUDNN( cudnnSetActivationDescriptor(activDesc,
(cudnnActivationMode_t) act_mode,
checkCUDNN(cudnnCreateActivationDescriptor(&activDesc));
checkCUDNN(cudnnSetActivationDescriptor(activDesc,
(cudnnActivationMode_t)act_mode,
CUDNN_PROPAGATE_NAN,
0.0) );
0.0));
}
}
Activation::~Activation() {
Activation::~Activation()
{
checkCuda( cudaFree(dstData) );
checkCuda(cudaFree(dstData));
if(int(act_mode) < 100)
checkCUDNN( cudnnDestroyActivationDescriptor(activDesc) );
if (int(act_mode) < 100)
checkCUDNN(cudnnDestroyActivationDescriptor(activDesc));
}
dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) {
dnnType *Activation::infer(dataDim_t &dim, dnnType *srcData)
{
if(act_mode == ACTIVATION_LEAKY) {
if (act_mode == ACTIVATION_LEAKY)
{
activationLEAKYForward(srcData, dstData, dim.tot());
} else {
}
else
{
dnnType alpha = dnnType(1);
dnnType beta = dnnType(0);
checkCUDNN( cudnnActivationForward(net->cudnnHandle,
activDesc,
&alpha,
srcTensorDesc,
srcData,
&beta,
dstTensorDesc,
dstData) );
}
dnnType beta = dnnType(0);
checkCUDNN(cudnnActivationForward(net->cudnnHandle,
activDesc,
&alpha,
srcTensorDesc,
srcData,
&beta,
dstTensorDesc,
dstData));
}
return dstData;
}
}}
} // namespace dnn
} // namespace tk
src/Conv2d.cpp
+77 -69
View File
@@ -2,14 +2,18 @@
#include "Layer.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
int strideH, int strideW, int paddingH, int paddingW,
const char* fname_weights, bool batchnorm) :
LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1,
fname_weights, batchnorm) {
Conv2d::Conv2d(Network *net, int out_ch, int kernelH, int kernelW,
int strideH, int strideW, int paddingH, int paddingW,
const char *fname_weights, bool batchnorm) :
LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1,
fname_weights, batchnorm)
{
this->kernelH = kernelH;
this->kernelW = kernelW;
@@ -18,56 +22,55 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
this->paddingH = paddingH;
this->paddingW = paddingW;
checkCUDNN( cudnnCreateFilterDescriptor(&filterDesc) );
checkCUDNN( cudnnCreateConvolutionDescriptor(&convDesc) );
checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) );
checkCUDNN(cudnnCreateFilterDescriptor(&filterDesc));
checkCUDNN(cudnnCreateConvolutionDescriptor(&convDesc));
checkCUDNN(cudnnCreateTensorDescriptor(&biasTensorDesc));
int n = input_dim.n;
int c = input_dim.c;
int h = input_dim.h;
int w = input_dim.w;
checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc,
net->tensorFormat, net->dataType, n, c, h, w) );
checkCUDNN(cudnnSetTensor4dDescriptor(srcTensorDesc,
net->tensorFormat, net->dataType, n, c, h, w));
checkCUDNN( cudnnSetFilter4dDescriptor(filterDesc,
net->dataType, net->tensorFormat, out_ch, input_dim.c,
kernelH, kernelW) );
checkCUDNN(cudnnSetFilter4dDescriptor(filterDesc,
net->dataType, net->tensorFormat, out_ch, input_dim.c,
kernelH, kernelW));
checkCUDNN( cudnnSetConvolution2dDescriptor(convDesc,
paddingH, paddingW, // padding
strideH, strideW, // stride
1,1, // upscale
CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT) );
checkCUDNN(cudnnSetConvolution2dDescriptor(convDesc,
paddingH, paddingW, // padding
strideH, strideW, // stride
1, 1, // upscale
CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT));
// find dimension of convolution output
checkCUDNN( cudnnGetConvolution2dForwardOutputDim(
convDesc, srcTensorDesc, filterDesc,
&n, &c, &h, &w) );
checkCUDNN(cudnnGetConvolution2dForwardOutputDim(
convDesc, srcTensorDesc, filterDesc,
&n, &c, &h, &w));
checkCUDNN(cudnnSetTensor4dDescriptor(dstTensorDesc,
net->tensorFormat, net->dataType, n, c, h, w));
checkCUDNN(cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle,
srcTensorDesc, filterDesc, convDesc, dstTensorDesc,
CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo));
checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
net->tensorFormat, net->dataType, n, c, h, w) );
checkCUDNN( cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle,
srcTensorDesc, filterDesc, convDesc, dstTensorDesc,
CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo) );
workSpace = NULL;
ws_sizeInBytes = 0;
checkCUDNN( cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle,
srcTensorDesc, filterDesc, convDesc, dstTensorDesc,
algo, &ws_sizeInBytes) );
checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle,
srcTensorDesc, filterDesc, convDesc, dstTensorDesc,
algo, &ws_sizeInBytes));
if (ws_sizeInBytes!=0) {
checkCuda( cudaMalloc(&workSpace, ws_sizeInBytes) );
if (ws_sizeInBytes != 0)
{
checkCuda(cudaMalloc(&workSpace, ws_sizeInBytes));
}
checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc,
net->tensorFormat, net->dataType,
1, out_ch, 1, 1) );
checkCUDNN(cudnnSetTensor4dDescriptor(biasTensorDesc,
net->tensorFormat, net->dataType,
1, out_ch, 1, 1));
output_dim.n = n;
output_dim.c = c;
@@ -76,53 +79,58 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
output_dim.l = 1;
//allocate data for infer result
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
checkCuda(cudaMalloc(&dstData, output_dim.tot() * sizeof(dnnType)));
}
Conv2d::~Conv2d() {
checkCUDNN( cudnnDestroyFilterDescriptor(filterDesc) );
checkCUDNN( cudnnDestroyConvolutionDescriptor(convDesc) );
checkCUDNN( cudnnDestroyTensorDescriptor(biasTensorDesc) );
Conv2d::~Conv2d()
{
if (ws_sizeInBytes!=0)
checkCuda( cudaFree(workSpace) );
checkCUDNN(cudnnDestroyFilterDescriptor(filterDesc));
checkCUDNN(cudnnDestroyConvolutionDescriptor(convDesc));
checkCUDNN(cudnnDestroyTensorDescriptor(biasTensorDesc));
checkCuda( cudaFree(dstData) );
if (ws_sizeInBytes != 0)
checkCuda(cudaFree(workSpace));
checkCuda(cudaFree(dstData));
}
dnnType* Conv2d::infer(dataDim_t &dim, dnnType* srcData) {
dnnType *Conv2d::infer(dataDim_t &dim, dnnType *srcData)
{
// convolution
dnnType alpha = dnnType(1);
dnnType beta = dnnType(0);
checkCUDNN( cudnnConvolutionForward(net->cudnnHandle,
&alpha, srcTensorDesc, srcData, filterDesc,
data_d, convDesc, algo, workSpace, ws_sizeInBytes,
&beta, dstTensorDesc, dstData) );
dnnType beta = dnnType(0);
checkCUDNN(cudnnConvolutionForward(net->cudnnHandle,
&alpha, srcTensorDesc, srcData, filterDesc,
data_d, convDesc, algo, workSpace, ws_sizeInBytes,
&beta, dstTensorDesc, dstData));
if(!batchnorm) {
if (!batchnorm)
{
// bias
alpha = dnnType(1);
beta = dnnType(1);
checkCUDNN( cudnnAddTensor(net->cudnnHandle,
&alpha, biasTensorDesc, bias_d,
&beta, dstTensorDesc, dstData) );
} else {
beta = dnnType(1);
checkCUDNN(cudnnAddTensor(net->cudnnHandle,
&alpha, biasTensorDesc, bias_d,
&beta, dstTensorDesc, dstData));
}
else
{
float one = 1;
float zero = 0;
cudnnBatchNormalizationForwardInference(net->cudnnHandle,
CUDNN_BATCHNORM_SPATIAL, &one, &zero,
dstTensorDesc, dstData, dstTensorDesc,
dstData, biasTensorDesc, //same tensor descriptor as bias
scales_d, bias_d, mean_d, variance_d,
CUDNN_BN_MIN_EPSILON);
CUDNN_BATCHNORM_SPATIAL, &one, &zero,
dstTensorDesc, dstData, dstTensorDesc,
dstData, biasTensorDesc, //same tensor descriptor as bias
scales_d, bias_d, mean_d, variance_d,
CUDNN_BN_MIN_EPSILON);
}
//update data dimensions
//update data dimensions
dim = output_dim;
return dstData;
}
}}
} // namespace dnn
} // namespace tk
src/Dense.cpp
+27 -21
View File
@@ -2,10 +2,13 @@
#include "Layer.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
Dense::Dense(Network *net, int out_ch, const char* fname_weights) :
LayerWgs(net, net->getOutputDim().tot(), out_ch, 1, 1, 1, fname_weights) {
Dense::Dense(Network *net, int out_ch, const char *fname_weights) : LayerWgs(net, net->getOutputDim().tot(), out_ch, 1, 1, 1, fname_weights)
{
output_dim.n = 1;
output_dim.c = out_ch;
@@ -14,19 +17,21 @@ Dense::Dense(Network *net, int out_ch, const char* fname_weights) :
output_dim.l = 1;
//allocate data for infer result
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
checkCuda(cudaMalloc(&dstData, output_dim.tot() * sizeof(dnnType)));
}
Dense::~Dense() {
Dense::~Dense()
{
checkCuda( cudaFree(dstData) );
checkCuda(cudaFree(dstData));
}
dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) {
dnnType *Dense::infer(dataDim_t &dim, dnnType *srcData)
{
if (dim.n != 1)
FatalError("Not Implemented");
FatalError("Not Implemented");
int dim_x = dim.tot();
int dim_y = output_dim.tot();
@@ -35,18 +40,18 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) {
dnnType alpha = dnnType(1), beta = dnnType(1);
// place bias into dstData
checkCuda( cudaMemcpy(dstData, bias_d, dim_y*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
//do matrix moltiplication
checkERROR( cublasSgemv(net->cublasHandle, CUBLAS_OP_T,
dim_x, dim_y,
&alpha,
data_d, dim_x,
srcData, 1,
&beta,
dstData, 1) );
checkCuda(cudaMemcpy(dstData, bias_d, dim_y * sizeof(dnnType), cudaMemcpyDeviceToDevice));
//update data dimensions
//do matrix moltiplication
checkERROR(cublasSgemv(net->cublasHandle, CUBLAS_OP_T,
dim_x, dim_y,
&alpha,
data_d, dim_x,
srcData, 1,
&beta,
dstData, 1));
//update data dimensions
dim.h = 1;
dim.w = 1;
dim.l = 1;
@@ -55,4 +60,5 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) {
return dstData;
}
}}
} // namespace dnn
} // namespace tk
src/Flatten.cpp
+16 -10
View File
@@ -3,34 +3,40 @@
#include "Layer.h"
#include "kernels.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
Flatten::Flatten(Network *net) : Layer(net) {
Flatten::Flatten(Network *net) : Layer(net)
{
checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
checkCuda(cudaMalloc(&dstData, input_dim.tot() * sizeof(dnnType)));
output_dim.n = 1;
output_dim.c = input_dim.tot();
output_dim.h = 1;
output_dim.w = 1;
output_dim.l = 1;
}
Flatten::~Flatten() {
Flatten::~Flatten()
{
checkCuda( cudaFree(dstData) );
checkCuda(cudaFree(dstData));
}
dnnType* Flatten::infer(dataDim_t &dim, dnnType* srcData) {
dnnType *Flatten::infer(dataDim_t &dim, dnnType *srcData)
{
//transpose per channel
matrixTranspose(net->cublasHandle, srcData, dstData, dim.c, dim.h*dim.w*dim.l);
matrixTranspose(net->cublasHandle, srcData, dstData, dim.c, dim.h * dim.w * dim.l);
//update data dimensions
//update data dimensions
dim = output_dim;
return dstData;
}
}}
} // namespace dnn
} // namespace tk
src/Layer.cpp
+19 -12
View File
@@ -2,28 +2,35 @@
#include "Layer.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
Layer::Layer(Network *net) {
Layer::Layer(Network *net)
{
this->net = net;
if(net != nullptr) {
if (net != nullptr)
{
this->input_dim = net->getOutputDim();
this->output_dim = input_dim;
checkCUDNN( cudnnCreateTensorDescriptor(&srcTensorDesc) );
checkCUDNN( cudnnCreateTensorDescriptor(&dstTensorDesc) );
if(!net->addLayer(this))
FatalError("Net reached max number of layers");
checkCUDNN(cudnnCreateTensorDescriptor(&srcTensorDesc));
checkCUDNN(cudnnCreateTensorDescriptor(&dstTensorDesc));
if (!net->addLayer(this))
FatalError("Net reached max number of layers");
}
}
Layer::~Layer() {
Layer::~Layer()
{
checkCUDNN( cudnnDestroyTensorDescriptor(srcTensorDesc) );
checkCUDNN( cudnnDestroyTensorDescriptor(dstTensorDesc) );
checkCUDNN(cudnnDestroyTensorDescriptor(srcTensorDesc));
checkCUDNN(cudnnDestroyTensorDescriptor(dstTensorDesc));
}
}}
} // namespace dnn
} // namespace tk
src/LayerWgs.cpp
+63 -54
View File
@@ -4,24 +4,29 @@
#include "Layer.h"
#include "kernels.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
int kh, int kw, int kl,
const char* fname_weights, bool batchnorm) : Layer(net) {
LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
int kh, int kw, int kl,
const char *fname_weights, bool batchnorm) : Layer(net)
{
this->inputs = inputs;
this->outputs = outputs;
this->weights_path = std::string(fname_weights);
std::cout<<"Reading weights: I="<<inputs<<" O="<<outputs<<" KERNEL="<<kh<<"x"<<kw<<"x"<<kl<<"\n";
this->inputs = inputs;
this->outputs = outputs;
this->weights_path = std::string(fname_weights);
std::cout << "Reading weights: I=" << inputs << " O=" << outputs << " KERNEL=" << kh << "x" << kw << "x" << kl << "\n";
int seek = 0;
readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d, seek);
seek += inputs*outputs*kh*kw*kl;
readBinaryFile(weights_path.c_str(), inputs * outputs * kh * kw * kl, &data_h, &data_d, seek);
seek += inputs * outputs * kh * kw * kl;
readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek);
this->batchnorm = batchnorm;
if(batchnorm) {
if (batchnorm)
{
seek += outputs;
readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek);
seek += outputs;
@@ -32,86 +37,90 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
float eps = CUDNN_BN_MIN_EPSILON;
power_h = new dnnType[outputs];
for(int i=0; i<outputs; i++) power_h[i] = 1.0f;
for (int i = 0; i < outputs; i++)
power_h[i] = 1.0f;
for(int i=0; i<outputs; i++)
mean_h[i] = mean_h[i] / -sqrt(eps + variance_h[i]);
for (int i = 0; i < outputs; i++)
mean_h[i] = mean_h[i] / -sqrt(eps + variance_h[i]);
for(int i=0; i<outputs; i++)
for (int i = 0; i < outputs; i++)
variance_h[i] = 1.0f / sqrt(eps + variance_h[i]);
}
if(!net->fp16)
if (!net->fp16)
return;
//convert to fp16
int w_size = inputs*outputs*kh*kw*kl;
int w_size = inputs * outputs * kh * kw * kl;
data16_h = new __half[w_size];
cudaMalloc(&data16_d, w_size*sizeof(__half));
cudaMalloc(&data16_d, w_size * sizeof(__half));
float2half(data_d, data16_d, w_size);
cudaMemcpy(data16_h, data16_d, w_size*sizeof(__half), cudaMemcpyDeviceToHost);
cudaMemcpy(data16_h, data16_d, w_size * sizeof(__half), cudaMemcpyDeviceToHost);
int b_size = outputs;
bias16_h = new __half[b_size];
cudaMalloc(&bias16_d, w_size*sizeof(__half));
cudaMalloc(&bias16_d, w_size * sizeof(__half));
float2half(bias_d, bias16_d, b_size);
cudaMemcpy(bias16_h, bias16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
cudaMemcpy(bias16_h, bias16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
if(batchnorm) {
if (batchnorm)
{
power16_h = new __half[b_size];
mean16_h = new __half[b_size];
power16_h = new __half[b_size];
mean16_h = new __half[b_size];
variance16_h = new __half[b_size];
scales16_h = new __half[b_size];
scales16_h = new __half[b_size];
cudaMalloc(&power16_d, b_size*sizeof(__half));
cudaMalloc(&mean16_d, b_size*sizeof(__half));
cudaMalloc(&variance16_d, b_size*sizeof(__half));
cudaMalloc(&scales16_d, b_size*sizeof(__half));
cudaMalloc(&power16_d, b_size * sizeof(__half));
cudaMalloc(&mean16_d, b_size * sizeof(__half));
cudaMalloc(&variance16_d, b_size * sizeof(__half));
cudaMalloc(&scales16_d, b_size * sizeof(__half));
//temporary buffers
float *tmp_d;
cudaMalloc(&tmp_d, b_size*sizeof(float));
cudaMalloc(&tmp_d, b_size * sizeof(float));
//init power array of ones
cudaMemcpy(tmp_d, power_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
cudaMemcpy(tmp_d, power_h, b_size * sizeof(float), cudaMemcpyHostToDevice);
float2half(tmp_d, power16_d, b_size);
cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
cudaMemcpy(power16_h, power16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
//mean array
cudaMemcpy(tmp_d, mean_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
cudaMemcpy(tmp_d, mean_h, b_size * sizeof(float), cudaMemcpyHostToDevice);
float2half(tmp_d, mean16_d, b_size);
cudaMemcpy(mean16_h, mean16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
cudaMemcpy(mean16_h, mean16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
//convert variance
cudaMemcpy(tmp_d, variance_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
cudaMemcpy(tmp_d, variance_h, b_size * sizeof(float), cudaMemcpyHostToDevice);
float2half(tmp_d, variance16_d, b_size);
cudaMemcpy(variance16_h, variance16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
cudaMemcpy(variance16_h, variance16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
//conver scales
float2half(scales_d, scales16_d, b_size);
cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
cudaMemcpy(scales16_h, scales16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
}
}
LayerWgs::~LayerWgs() {
LayerWgs::~LayerWgs()
{
delete [] data_h;
delete [] bias_h;
checkCuda( cudaFree(data_d) );
checkCuda( cudaFree(bias_d) );
delete[] data_h;
delete[] bias_h;
checkCuda(cudaFree(data_d));
checkCuda(cudaFree(bias_d));
if(batchnorm) {
delete [] scales_h;
delete [] mean_h;
delete [] variance_h;
checkCuda( cudaFree(scales_d) );
checkCuda( cudaFree(mean_d) );
checkCuda( cudaFree(variance_d) );
if (batchnorm)
{
delete[] scales_h;
delete[] mean_h;
delete[] variance_h;
checkCuda(cudaFree(scales_d));
checkCuda(cudaFree(mean_d));
checkCuda(cudaFree(variance_d));
}
}
}}
} // namespace dnn
} // namespace tk
src/MulAdd.cpp
+22 -16
View File
@@ -3,42 +3,48 @@
#include "Layer.h"
#include "kernels.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) {
MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net)
{
this->mul = mul;
this->add = add;
int size = input_dim.tot();
// create a vector with all value setted to add
// create a vector with all value setted to add
dnnType *add_vector_h = new dnnType[size];
for(int i=0; i<size; i++)
for (int i = 0; i < size; i++)
add_vector_h[i] = add;
checkCuda( cudaMalloc(&add_vector, size*sizeof(dnnType)));
checkCuda( cudaMemcpy(add_vector, add_vector_h, size*sizeof(dnnType), cudaMemcpyHostToDevice));
delete [] add_vector_h;
checkCuda(cudaMalloc(&add_vector, size * sizeof(dnnType)));
checkCuda(cudaMemcpy(add_vector, add_vector_h, size * sizeof(dnnType), cudaMemcpyHostToDevice));
delete[] add_vector_h;
checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
checkCuda(cudaMalloc(&dstData, input_dim.tot() * sizeof(dnnType)));
}
MulAdd::~MulAdd() {
MulAdd::~MulAdd()
{
checkCuda( cudaFree(add_vector) );
checkCuda( cudaFree(dstData) );
checkCuda(cudaFree(add_vector));
checkCuda(cudaFree(dstData));
}
dnnType* MulAdd::infer(dataDim_t &dim, dnnType* srcData) {
dnnType *MulAdd::infer(dataDim_t &dim, dnnType *srcData)
{
matrixMulAdd(net->cublasHandle, srcData, dstData, add_vector, input_dim.tot(), mul);
//update data dimensions
//update data dimensions
dim = output_dim;
return dstData;
}
}}
} // namespace dnn
} // namespace tk
src/Network.cpp
+83 -58
View File
@@ -5,106 +5,131 @@
#include "Network.h"
#include "Layer.h"
namespace tk { namespace dnn {
namespace tk
{
namespace dnn
{
Network::Network(dataDim_t input_dim) {
Network::Network(dataDim_t input_dim)
{
this->input_dim = input_dim;
float tk_ver = float(TKDNN_VERSION)/1000;
float cu_ver = float(cudnnGetVersion())/1000;
float tk_ver = float(TKDNN_VERSION) / 1000;
float cu_ver = float(cudnnGetVersion()) / 1000;
std::cout<<"New NETWORK (tkDNN v"<<tk_ver
<<", CUDNN v"<<cu_ver<<")\n";
std::cout << "New NETWORK (tkDNN v" << tk_ver
<< ", CUDNN v" << cu_ver << ")\n";
dataType = CUDNN_DATA_FLOAT;
tensorFormat = CUDNN_TENSOR_NCHW;
checkCUDNN( cudnnCreate(&cudnnHandle) );
checkERROR( cublasCreate(&cublasHandle) );
checkCUDNN(cudnnCreate(&cudnnHandle));
checkERROR(cublasCreate(&cublasHandle));
num_layers = 0;
fp16 = false;
dla = false;
if(const char* env_p = std::getenv("TKDNN_MODE")) {
if(strcmp(env_p, "FP16") == 0)
if (const char *env_p = std::getenv("TKDNN_MODE"))
{
if (strcmp(env_p, "FP16") == 0)
fp16 = true;
else if(strcmp(env_p, "DLA") == 0) {
dla = true;
fp16 = true;
}
else if (strcmp(env_p, "DLA") == 0)
{
dla = true;
fp16 = true;
}
}
if(fp16)
std::cout<<COL_REDB<<"!! FP16 INERENCE ENABLED !!"<<COL_END<<"\n";
if(dla)
std::cout<<COL_GREENB<<"!! DLA INERENCE ENABLED !!"<<COL_END<<"\n";
if (fp16)
std::cout << COL_REDB << "!! FP16 INERENCE ENABLED !!" << COL_END << "\n";
if (dla)
std::cout << COL_GREENB << "!! DLA INERENCE ENABLED !!" << COL_END << "\n";
}
Network::~Network() {
Network::~Network()
{
checkCUDNN( cudnnDestroy(cudnnHandle) );
checkERROR( cublasDestroy(cublasHandle) );
checkCUDNN(cudnnDestroy(cudnnHandle));
checkERROR(cublasDestroy(cublasHandle));
}
dnnType* Network::infer(dataDim_t &dim, dnnType* data) {
dnnType *Network::infer(dataDim_t &dim, dnnType *data)
{
//do infer for every layer
for(int i=0; i<num_layers; i++) {
for (int i = 0; i < num_layers; i++)
{
data = layers[i]->infer(dim, data);
}
checkCuda(cudaDeviceSynchronize());
return data;
}
bool Network::addLayer(Layer *l) {
if(num_layers == MAX_LAYERS)
bool Network::addLayer(Layer *l)
{
if (num_layers == MAX_LAYERS)
return false;
layers[num_layers++] = l;
return true;
}
dataDim_t Network::getOutputDim() {
dataDim_t Network::getOutputDim()
{
if(num_layers == 0)
return input_dim;
else
return layers[num_layers-1]->output_dim;
if (num_layers == 0)
return input_dim;
else
return layers[num_layers - 1]->output_dim;
}
void Network::print() {
void Network::print()
{
printCenteredTitle(" NETWORK MODEL ", '=', 60);
std::cout.width(3); std::cout<<std::left<<"N.";
std::cout<<" ";
std::cout.width(17); std::cout<<std::left<<"Layer type";
std::cout.width(22); std::cout<<std::left<<"input (H*W,CH)";
std::cout.width(16); std::cout<<std::left<<"output (H*W,CH)";
std::cout<<"\n";
std::cout.width(3);
std::cout << std::left << "N.";
std::cout << " ";
std::cout.width(17);
std::cout << std::left << "Layer type";
std::cout.width(22);
std::cout << std::left << "input (H*W,CH)";
std::cout.width(16);
std::cout << std::left << "output (H*W,CH)";
std::cout << "\n";
for(int i=0; i<num_layers; i++) {
for (int i = 0; i < num_layers; i++)
{
dataDim_t in = layers[i]->input_dim;
dataDim_t out = layers[i]->output_dim;
std::cout.width(3); std::cout<<std::right<<i;
std::cout<<" ";
std::cout.width(16); std::cout<<std::left<<layers[i]->getLayerName();
std::cout.width(4); std::cout<<std::right<<in.h;
std::cout<<" x ";
std::cout.width(4); std::cout<<std::right<<in.w;
std::cout<<", ";
std::cout.width(4); std::cout<<std::right<<in.c;
std::cout<<" -> ";
std::cout.width(4); std::cout<<std::right<<out.h;
std::cout<<" x ";
std::cout.width(4); std::cout<<std::right<<out.w;
std::cout<<", ";
std::cout.width(4); std::cout<<std::right<<out.c;
std::cout<<"\n";
std::cout.width(3);
std::cout << std::right << i;
std::cout << " ";
std::cout.width(16);
std::cout << std::left << layers[i]->getLayerName();
std::cout.width(4);
std::cout << std::right << in.h;
std::cout << " x ";
std::cout.width(4);
std::cout << std::right << in.w;
std::cout << ", ";
std::cout.width(4);
std::cout << std::right << in.c;
std::cout << " -> ";
std::cout.width(4);
std::cout << std::right << out.h;
std::cout << " x ";
std::cout.width(4);
std::cout << std::right << out.w;
std::cout << ", ";
std::cout.width(4);
std::cout << std::right << out.c;
std::cout << "\n";
}
printCenteredTitle("", '=', 60);
std::cout<<"\n";
std::cout << "\n";
}
}}
} // namespace dnn
} // namespace tk
src/BoxDetection.cpp → src/class_src/boxDetection.cpp
+177 -158
View File
@@ -1,6 +1,6 @@
#include "BoxDetection.h"
#include "boxDetection.h"
#include <string.h>
char buf_frame_crop_name [200];
char buf_frame_crop_name[200];
cv::Mat img_threshold(cv::Mat frame_crop)
{
@@ -24,10 +24,10 @@ cv::Mat img_background(cv::Mat frame_crop)
cv::cvtColor(f, gray, cv::COLOR_RGBA2GRAY, 0);
cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU);
// get background
cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1,1,1,1));
cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1, 1, 1, 1));
cv::erode(gray, opening, M);
cv::dilate(gray, opening, M);
cv::Point p = cv::Point(-1,-1);
cv::Point p = cv::Point(-1, -1);
cv::dilate(opening, coinsBg, M, p, 3);
return coinsBg;
}
@@ -43,10 +43,10 @@ cv::Mat img_dist_transform(cv::Mat frame_crop)
cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU);
// cv::Mat::ones M(3,3,cv::CV_8U);
// get background
cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1,1,1,1));
cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1, 1, 1, 1));
cv::erode(gray, opening, M);
cv::dilate(gray, opening, M);
cv::Point p = cv::Point(-1,-1);
cv::Point p = cv::Point(-1, -1);
cv::dilate(opening, coinsBg, M, p, 3);
// distance transorm
cv::distanceTransform(opening, distTrans, cv::DIST_L2, 5);
@@ -75,11 +75,11 @@ cv::Mat img_dist_transform(cv::Mat frame_crop)
// // get foreground
// cv::threshold(distTrans, coinsFg, 0.7 * 1, 255, cv::THRESH_BINARY);
// coinsFg.convertTo(coinsFg, CV_8U, 1, 0);
// coinsFg.convertTo(coinsFg, CV_8U, 1, 0);
// cv::subtract(coinsBg, coinsFg, unknown);
// // get connected components networks
// cv::connectedComponents(coinsFg, markers);
// // intptr_t n = NULL;
// // intptr_t n = NULL;
// for(int i = 0; i< markers.rows; i++)
// {
// for (int j = 0; j< markers.cols; j++)
@@ -111,7 +111,7 @@ cv::Mat img_dist_transform(cv::Mat frame_crop)
//////
cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret=0)
cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret = 0)
{
//ret = 0 --> dstx
//ret = 1 --> dsty
@@ -120,61 +120,61 @@ cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret=0)
// Image Sobel and Image AbsSobel
// https://docs.opencv.org/trunk/da/d85/tutorial_js_gradients.html
// compute image gradient on two different directions
cv::Mat f = frame_crop.clone();
int x,y;
(ret == 0 || ret == 2)?x=1, y=0 : NULL;
(ret == 1 || ret == 3)?x=0, y=1 : NULL;
cv::Mat dst;
int x, y;
(ret == 0 || ret == 2) ? x = 1, y = 0 : NULL;
(ret == 1 || ret == 3) ? x = 0, y = 1 : NULL;
cv::Mat dst;
cv::cvtColor(f, f, cv::COLOR_RGB2GRAY, 0);
// You can try more different parameters
cv::Sobel(f, dst, CV_8U, x, y, 3, 1, 0, cv::BORDER_DEFAULT);
// for absSobel
if(ret == 2 || ret == 3)
if (ret == 2 || ret == 3)
cv::convertScaleAbs(dst, dst, 1, 0);
// next 3 rows to be checked
//// ??cv::Mat f2 = frame_crop.clone();
//// cv.Scharr(?(f,f2), dstx, cv.CV_8U, 1, 0, 1, 0, cv.BORDER_DEFAULT);
//// cv.Scharr(?(f,f2), dsty, cv.CV_8U, 0, 1, 1, 0, cv.BORDER_DEFAULT);
//// cv.Scharr(?(f,f2), dsty, cv.CV_8U, 0, 1, 1, 0, cv.BORDER_DEFAULT);
return dst;
}
cv::Mat img_laplacian(cv::Mat frame_crop, int ret=1)
cv::Mat img_laplacian(cv::Mat frame_crop, int ret = 1)
{
//ret = 0 --> src_gray
//ret = 1 --> dst
// Image Laplacian
// compute image gradient with laplacian
// compute image gradient with laplacian
cv::Mat f = frame_crop.clone();
cv::Mat src_gray, dst;
int kernel_size = 3;
int scale = 1;
int delta = 0;
int ddepth = CV_16S;
cv::GaussianBlur( f, f, cv::Size(3,3), 0, 0, cv::BORDER_DEFAULT );
cv::GaussianBlur(f, f, cv::Size(3, 3), 0, 0, cv::BORDER_DEFAULT);
/// Convert the image to grayscale
cv::cvtColor( f, src_gray, CV_RGB2GRAY );
cv::cvtColor(f, src_gray, CV_RGB2GRAY);
if (ret == 0)
return src_gray;
// else: Apply Laplace function
cv::Mat abs_dst;
cv::Laplacian( src_gray, dst, ddepth, kernel_size, scale, delta, cv::BORDER_DEFAULT );
cv::Laplacian(src_gray, dst, ddepth, kernel_size, scale, delta, cv::BORDER_DEFAULT);
// //compute sharpness
// float sharpnessValue = cv::mean(dst);
return dst;
return dst;
}
cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int n_lines=1)
cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int n_lines = 1)
{
// n_line: number of line to plot on image
cv::Mat img_line = frame_crop.clone();
cv::Mat ret_thresh;
std::vector<std::vector<cv::Point> > contours;
std::vector<std::vector<cv::Point>> contours;
double thresh = 127;
double maxValue = 255;
cv::threshold(img, ret_thresh, thresh, maxValue, 0);//0); // = cv2.threshold(img,127,255,0)
cv::findContours(canny_output, contours, 1, 2);//cv::CHAIN_APPROX_SIMPLE );//1, 2); //contours,hierarchy = cv2.findContours(thresh, 1, 2)
cv::threshold(img, ret_thresh, thresh, maxValue, 0); //0); // = cv2.threshold(img,127,255,0)
cv::findContours(canny_output, contours, 1, 2); //cv::CHAIN_APPROX_SIMPLE );//1, 2); //contours,hierarchy = cv2.findContours(thresh, 1, 2)
// cv::threshold(img2, ret2, thresh, maxValue, 0);
// cv::findContours(canny_output2, contours2, 1, 2);
// cv::threshold(img3a, ret3a, thresh, maxValue, 0);
@@ -183,23 +183,23 @@ cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int
// cv::findContours(canny_output3b, contours3b, 1, 2);
cv::Vec4f line;
float vx,vy,x,y;
float vx, vy, x, y;
int lefty, righty;
for(int i=0; i<n_lines; i++)
for (int i = 0; i < n_lines; i++)
{
cv::fitLine(contours[i],line,CV_DIST_L2,0,0.01,0.01);
cv::fitLine(contours[i], line, CV_DIST_L2, 0, 0.01, 0.01);
vx = line(0);
vy = line(1);
x = line(2);
y = line(3);
lefty = int((-x*vy/vx) + y);
righty = int(((img.cols-x)*vy/vx)+y);
cv::line(img_line,cv::Point(img.cols-1,righty),cv::Point(0,lefty),(255, 0 ,0),2);
y = line(3);
lefty = int((-x * vy / vx) + y);
righty = int(((img.cols - x) * vy / vx) + y);
cv::line(img_line, cv::Point(img.cols - 1, righty), cv::Point(0, lefty), (255, 0, 0), 2);
}
// cv::imshow("bla", img);
// cv::waitKey(1000);
return img_line;
return img_line;
}
// cv::Mat fit_rectangular(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output)
@@ -215,7 +215,7 @@ cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int
// cv::RotatedRect rect = cv::minAreaRect(contours[0]);
// cv::Mat boxPts1;
// std::vector<std::vector<cv::Point> > boxPts2;
// cv::boxPoints(rect, boxPts1);
// cv::boxPoints(rect, boxPts1);
// // boxPts = np.int0(boxPts);
// for (int x = 0; x < img.cols; x++)
// for (int y = 0; y < img.rows; y++)
@@ -226,7 +226,7 @@ cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int
// return img_clone;
// }
cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr<cv::saliency::Saliency> saliencyAlgorithm, int const_molt_mat, int ret=0)
cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr<cv::saliency::Saliency> saliencyAlgorithm, int const_molt_mat, int ret = 0)
{
//ret=0 --> saliencyMap
//ret=1 --> binaryMap
@@ -234,110 +234,117 @@ cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr<cv::saliency::Saliency> sa
cv::Mat f = frame_crop.clone();
cv::Mat saliencyMap;
cv::Mat binaryMap;
if( saliencyAlgorithm->computeSaliency( f, saliencyMap ) )
if (saliencyAlgorithm->computeSaliency(f, saliencyMap))
{
if(ret==0)
return saliencyMap*const_molt_mat;
if (ret == 0)
return saliencyMap * const_molt_mat;
cv::saliency::StaticSaliencySpectralResidual spec;
spec.computeBinaryMap( saliencyMap, binaryMap );
spec.computeBinaryMap(saliencyMap, binaryMap);
// imshow( "Saliency Map", saliencyMap );
// imshow( "Original Image", image );
// imshow( "Binary Map", binaryMap );
// waitKey( 0 );
return binaryMap*const_molt_mat;
return binaryMap * const_molt_mat;
}
return cv::Mat(0,0,CV_8U, cv::Scalar(0,0,0,0));
return cv::Mat(0, 0, CV_8U, cv::Scalar(0, 0, 0, 0));
}
//////
void image_segmentation(cv::Mat frame_crop, int frame_nbr, int i)
{
{
// Watershed Algorithm
// https://docs.opencv.org/3.4/d7/d1c/tutorial_js_watershed.html
auto step_t_segmentation = std::chrono::steady_clock::now();
auto end_t_segmentation = std::chrono::steady_clock::now();
cv::Mat ret;
// ret = img_threshold(frame_crop);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgthr.jpg", frame_nbr, i, img_threshold(frame_crop));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgthr.jpg", frame_nbr, i, img_threshold(frame_crop));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME imgthr ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME imgthr (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// ret =img_background(frame_crop);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgback.jpg", frame_nbr, i, img_background(frame_crop));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgback.jpg", frame_nbr, i, img_background(frame_crop));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME imgback ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME imgback (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// ret = img_dist_transform(frame_crop);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgtrans.jpg", frame_nbr, i, img_dist_transform(frame_crop));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgtrans.jpg", frame_nbr, i, img_dist_transform(frame_crop));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME imgtrans ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME imgtrans (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// // ret = img_watershed(frame_crop);
// if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgwatershed.jpg", frame_nbr, i, img_watershed(frame_crop));
// if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgwatershed.jpg", frame_nbr, i, img_watershed(frame_crop));
}
void image_gradients(cv::Mat frame_crop, int frame_nbr, int i)
{
// Image Gradients
// https://docs.opencv.org/trunk/da/d85/tutorial_js_gradients.html
auto step_t_segmentation = std::chrono::steady_clock::now();
auto end_t_segmentation = std::chrono::steady_clock::now();
cv::Mat ret;
// sobel
// ret = img_sobel_abssobel(frame_crop, 0);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_8U.jpgg", frame_nbr, i, img_sobel_abssobel(frame_crop, 0));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_8U.jpgg", frame_nbr, i, img_sobel_abssobel(frame_crop, 0));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME sobel0 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME sobel0 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// ret = img_sobel_abssobel(frame_crop, 1);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_8U.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 1));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_8U.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 1));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME sobel1 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME sobel1 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// ret = img_sobel_abssobel(frame_crop, 2);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 2));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 2));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME sobel2 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME sobel2 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// ret = img_sobel_abssobel(frame_crop, 3);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 3));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 3));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME sobel3 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME sobel3 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// laplacian
// ret = img_laplacian(frame_crop, 0);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_gr.jpg", frame_nbr, i, img_laplacian(frame_crop, 0));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_gr.jpg", frame_nbr, i, img_laplacian(frame_crop, 0));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME laplacian0 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
step_t_segmentation = end_t_segmentation;
std::cout << " - TIME laplacian0 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// ret = img_laplacian(frame_crop, 1);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_dst.jpg", frame_nbr, i, img_laplacian(frame_crop, 1));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_dst.jpg", frame_nbr, i, img_laplacian(frame_crop, 1));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME laplacian1 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME laplacian1 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
//////
}
}
void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i)
{
// Finding contours in your image
// https://docs.opencv.org/3.4/df/d0d/tutorial_find_contours.html
auto step_t_segmentation = std::chrono::steady_clock::now();
auto end_t_segmentation = std::chrono::steady_clock::now();
// plot lines on figure. 3 ways:
@@ -348,60 +355,67 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i)
cv::Mat contours;
// src_gray
cv::Mat img1 = img_laplacian(frame_crop, 0);
cv::Canny(img1, canny_output1, 100, 100*2 );
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny1.jpg", frame_nbr, i, canny_output1);
cv::Canny(img1, canny_output1, 100, 100 * 2);
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny1.jpg", frame_nbr, i, canny_output1);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME canny1 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME canny1 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// // dst
// cv::Mat img2 = img_laplacian(frame_crop, 2);
// cv::Canny(img2, canny_output2, 100, 100*2 );
cv::Canny(img1, canny_output2, 100, 100*2 );
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny2.jpg", frame_nbr, i, canny_output2);
cv::Canny(img1, canny_output2, 100, 100 * 2);
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny2.jpg", frame_nbr, i, canny_output2);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME canny2 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME canny2 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// dstx
cv::Mat img3a = img_sobel_abssobel(frame_crop, 0);
cv::Canny(img3a, canny_output3a, 100, 100*2 );
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3a.jpg", frame_nbr, i, canny_output3a);
cv::Canny(img3a, canny_output3a, 100, 100 * 2);
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3a.jpg", frame_nbr, i, canny_output3a);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME canny3a ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME canny3a (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// dsty
cv::Mat img3b = img_sobel_abssobel(frame_crop, 1);
cv::Canny(img3b, canny_output3b, 100, 100*2 );
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3b.jpg", frame_nbr, i, canny_output3b);
cv::Canny(img3b, canny_output3b, 100, 100 * 2);
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3b.jpg", frame_nbr, i, canny_output3b);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME canny3b ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME canny3b (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// 1 line
// contours = find_contours(frame_crop, img1, canny_output1, 1);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_line1.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output1, 1));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_line1.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output1, 1));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME line1 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME line1 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// 3 line
// contours = find_contours(frame_crop, img1, canny_output2, 1);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_line2.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output2, 1));
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_line2.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output2, 1));
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME line2 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME line2 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// mix 1 line of image with 1 line of another
cv::Mat img_line = frame_crop.clone();
img_line = find_contours(img_line, img3a, canny_output3a, 1);
img_line = find_contours(img_line, img3b, canny_output3b, 1);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_line3.jpg", frame_nbr, i, img_line);
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_line3.jpg", frame_nbr, i, img_line);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME line3 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME line3 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// img_line = frame_crop.clone();
@@ -409,8 +423,7 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i)
// img_line = find_contours(img_line, img3b, canny_output3b, 2);
// sprintf(buf_frame_crop_name,"../demo/demo/data/img_crop/%d_%d_line3bis.jpg",frame_nbr, i);
// cv::imwrite(buf_frame_crop_name, img_line);
// cv::Mat canny_output4;
// cv::Mat img4 = img_laplacian(frame_crop, 0);
// cv::Canny(img4, canny_output4, 100, 100*2 );
@@ -418,12 +431,11 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i)
// cv::imwrite(buf_frame_crop_name, fit_rectangular(frame_crop, img4, canny_output4));
}
void image_saliency(cv::Mat frame_crop, int frame_nbr, int i)
{
// https://github.com/opencv/opencv_contrib/blob/master/modules/saliency/samples/computeSaliency.cpp
cv::Ptr<cv::saliency::Saliency> saliencyAlgorithm;
int const_molt_mat = 0;
auto step_t_segmentation = std::chrono::steady_clock::now();
auto end_t_segmentation = std::chrono::steady_clock::now();
@@ -432,47 +444,50 @@ void image_saliency(cv::Mat frame_crop, int frame_nbr, int i)
const_molt_mat = 255;
saliencyAlgorithm = cv::saliency::StaticSaliencySpectralResidual::create();
cv::Mat spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 0);
if(!spect_res.empty())
if (!spect_res.empty())
{
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_SpectralResidual.jpg", frame_nbr, i, spect_res);
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_SpectralResidual.jpg", frame_nbr, i, spect_res);
}
else
{
std::cout<<"something is wrond (image_saliency)"<<std::endl;
std::cout << "something is wrond (image_saliency)" << std::endl;
}
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME SPECTRAL_RESIDUAL ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME SPECTRAL_RESIDUAL (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// BINARY SPECTRAL_RESIDUAL
const_molt_mat = 255;
spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 1);
if(!spect_res.empty())
if (!spect_res.empty())
{
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinarySpectralResidual.jpg", frame_nbr, i, spect_res);
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinarySpectralResidual.jpg", frame_nbr, i, spect_res);
}
else
{
std::cout<<"something is wrond (image_saliency)"<<std::endl;
std::cout << "something is wrond (image_saliency)" << std::endl;
}
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME BINARY SPECTRAL_RESIDUAL ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME BINARY SPECTRAL_RESIDUAL (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// FINE_GRAINED
const_molt_mat = 1;
saliencyAlgorithm = cv::saliency::StaticSaliencyFineGrained::create();
saliencyAlgorithm = cv::saliency::StaticSaliencyFineGrained::create();
spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 0);
if(!spect_res.empty())
if (!spect_res.empty())
{
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_FineGrained.jpg", frame_nbr, i, spect_res);
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_FineGrained.jpg", frame_nbr, i, spect_res);
}
else
{
std::cout<<"something is wrond (image_saliency)"<<std::endl;
std::cout << "something is wrond (image_saliency)" << std::endl;
}
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME FINE_GRAINED ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME FINE_GRAINED (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// saliencyAlgorithm = cv::saliency::ObjectnessBING::create();
@@ -486,7 +501,7 @@ void image_saliency(cv::Mat frame_crop, int frame_nbr, int i)
// // // The result are sorted by objectness. We only use the first maxd boxes here.
// // int maxd = 7, step = 255 / maxd, jitter=9; // jitter to seperate single rects
// // cv::Mat draw = frame_crop.clone();
// // for (int i = 0; i < std::min(maxd, ndet); i++)
// // for (int i = 0; i < std::min(maxd, ndet); i++)
// // {
// // cv::Vec4i bb = saliencyMap1[i];
// // cv::Scalar col = cv::Scalar(((i*step)%255), 100, 255-((i*step)%255));
@@ -499,34 +514,35 @@ void image_saliency(cv::Mat frame_crop, int frame_nbr, int i)
// printf(buf_frame_crop_name,"../demo/demo/data/img_crop/%d_%d_saliency_BING.jpg",frame_nbr, i);
// cv::imwrite(buf_frame_crop_name, saliencyMap1);
////
////
// BING WANG APR 2014
cv::Mat saliencyMap;
cv::Mat frame_sal = frame_crop.clone();
saliencyAlgorithm = cv::saliency::MotionSaliencyBinWangApr2014::create();
saliencyAlgorithm.dynamicCast<cv::saliency::MotionSaliencyBinWangApr2014>()->setImagesize( frame_sal.cols, frame_sal.rows );
saliencyAlgorithm.dynamicCast<cv::saliency::MotionSaliencyBinWangApr2014>()->setImagesize(frame_sal.cols, frame_sal.rows);
saliencyAlgorithm.dynamicCast<cv::saliency::MotionSaliencyBinWangApr2014>()->init();
cvtColor( frame_sal, frame_sal, cv::COLOR_BGR2GRAY );
saliencyAlgorithm->computeSaliency( frame_sal, saliencyMap);
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinWangApr.jpg", frame_nbr, i, saliencyMap);
cvtColor(frame_sal, frame_sal, cv::COLOR_BGR2GRAY);
saliencyAlgorithm->computeSaliency(frame_sal, saliencyMap);
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinWangApr.jpg", frame_nbr, i, saliencyMap);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " - TIME BING WANG APR 2014("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " - TIME BING WANG APR 2014(" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
}
cv::Mat frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i, int ret=0)
cv::Mat frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i, int ret = 0)
{
// https://stackoverflow.com/questions/27035672/cv-extract-differences-between-two-images
cv::Mat backgroundImage = pre_frame.clone();
cv::Mat currentImage = frame.clone();
cv::Mat diffImage;
// pass to HSV color
if(ret)
if (ret)
{
cv::cvtColor(backgroundImage, backgroundImage, CV_BGR2HSV);
cv::cvtColor(currentImage, currentImage, CV_BGR2HSV);
cv::cvtColor(backgroundImage, backgroundImage, CV_BGR2HSV);
cv::cvtColor(currentImage, currentImage, CV_BGR2HSV);
}
cv::absdiff(backgroundImage, currentImage, diffImage);
@@ -537,27 +553,28 @@ cv::Mat frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i,
float threshold = 30.0f;
float dist;
for(int j=0; j<diffImage.rows; ++j)
for (int j = 0; j < diffImage.rows; ++j)
{
for(int k=0; k<diffImage.cols; ++k)
for (int k = 0; k < diffImage.cols; ++k)
{
cv::Vec3b pix = diffImage.at<cv::Vec3b>(j,k);
cv::Vec3b pix = diffImage.at<cv::Vec3b>(j, k);
dist = (pix[0]*pix[0] + pix[1]*pix[1] + pix[2]*pix[2]);
dist = (pix[0] * pix[0] + pix[1] * pix[1] + pix[2] * pix[2]);
dist = sqrt(dist);
if(dist>threshold)
if (dist > threshold)
{
foregroundMask.at<unsigned char>(j,k) = 255;
foregroundMask.at<unsigned char>(j, k) = 255;
}
}
}
if(SAVE) SAVE_TO("../demo/demo/data/img_disparity/%d_%d_dif.jpg", frame_nbr, i, foregroundMask);
if (SAVE)
SAVE_TO("../demo/demo/data/img_disparity/%d_%d_dif.jpg", frame_nbr, i, foregroundMask);
return foregroundMask;
}
void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector <cv::Rect> pre_rois, int frame_nbr)
void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector<cv::Rect> pre_rois, int frame_nbr)
{
int roi_tollerance = 10;
@@ -567,35 +584,38 @@ void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector <cv::Rect
auto step_t_segmentation = std::chrono::steady_clock::now();
auto end_t_segmentation = std::chrono::steady_clock::now();
for(auto r : pre_rois)
for (auto r : pre_rois)
{
if(SAVE) SAVE_TO("../demo/demo/data/img_disparity/%d_%d_orig.jpg", frame_nbr, id, pre_frame(r));
if (SAVE)
SAVE_TO("../demo/demo/data/img_disparity/%d_%d_orig.jpg", frame_nbr, id, pre_frame(r));
//resize last roi with a tollerance
dx = r.width / roi_tollerance;
dy = r.height / roi_tollerance;
r.x = (r.x - dx > 0)? (r.x - dx) : 0;
r.y = (r.y - dy > 0)? (r.y - dy) : 0;
r.x = (r.x - dx > 0) ? (r.x - dx) : 0;
r.y = (r.y - dy > 0) ? (r.y - dy) : 0;
// std::cout<<"disp: x "<<r.x<<" - y "<<r.y<<std::endl;
r.width = ((r.x+r.width+dx+dx) >= frame.cols)? (frame.cols-1-r.x) : (r.width+dx+dx);
r.height = ((r.y+r.height+dy+dy) >= frame.rows)? (frame.rows-1-r.y) : (r.height+dy+dy);
r.width = ((r.x + r.width + dx + dx) >= frame.cols) ? (frame.cols - 1 - r.x) : (r.width + dx + dx);
r.height = ((r.y + r.height + dy + dy) >= frame.rows) ? (frame.rows - 1 - r.y) : (r.height + dy + dy);
// std::cout<<"disp: w "<<r.width<<" - h "<<r.height<<std::endl;
// std::cout<<"disp: wf "<<frame.cols<<" - hf "<<frame.rows<<std::endl;
// std::cout<<"---"<<std::endl;
// std::cout<<"disp: x "<<r.x<<" to "<<r.width+r.x<<" wf "<<frame.cols<<std::endl;
// std::cout<<"disp: y "<<r.y<<" to "<<r.height+r.y<<" hf "<<frame.rows<<std::endl;
//crop pre_frame and current frame
//crop pre_frame and current frame
pre_frame_crop = pre_frame(r);
frame_crop = frame(r);
if(SAVE) SAVE_TO("../demo/demo/data/img_disparity/%d_%d_cur.jpg", frame_nbr, id, frame_crop);
if(SAVE) SAVE_TO("../demo/demo/data/img_disparity/%d_%d_pre.jpg", frame_nbr, id, pre_frame_crop);
if (SAVE)
SAVE_TO("../demo/demo/data/img_disparity/%d_%d_cur.jpg", frame_nbr, id, frame_crop);
if (SAVE)
SAVE_TO("../demo/demo/data/img_disparity/%d_%d_pre.jpg", frame_nbr, id, pre_frame_crop);
// difference from two consecutive frame
step_t_segmentation = std::chrono::steady_clock::now();
frame_disparity(pre_frame_crop, frame_crop, frame_nbr, id, 0);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " TIME frame_disparity ("<<frame_nbr<<"-"<<id<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " TIME frame_disparity (" << frame_nbr << "-" << id << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
id++;
}
@@ -606,54 +626,53 @@ void segmentation(cv::Mat pre_frame, cv::Mat frame_crop, int frame_nbr, int i, i
//mode=0 (for whole frame), it computes the frame disparity
//mode=1 (for single box), it doesn't compute the frame disparity (it has already been done-see frame_box_disparity())
// whole figure
char buf_str [15];
if(!mode)
sprintf(buf_str,"whole frame");
char buf_str[15];
if (!mode)
sprintf(buf_str, "whole frame");
else
sprintf(buf_str,"a box frame");
if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d.jpg", frame_nbr, i, frame_crop);
sprintf(buf_str, "a box frame");
if (SAVE)
SAVE_TO("../demo/demo/data/img_crop/%d_%d.jpg", frame_nbr, i, frame_crop);
auto step_t_segmentation = std::chrono::steady_clock::now();
auto end_t_segmentation = std::chrono::steady_clock::now();
// Watershed Algorithm
std::cout<<"image segmentation:"<<std::endl;
std::cout << "image segmentation:" << std::endl;
image_segmentation(frame_crop, frame_nbr, i);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " TIME "<<buf_str<<": image_segmentation : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " TIME " << buf_str << ": image_segmentation : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// Image Gradients
std::cout<<"image gradients:"<<std::endl;
std::cout << "image gradients:" << std::endl;
image_gradients(frame_crop, frame_nbr, i);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " TIME "<<buf_str<<": image_gradients : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " TIME " << buf_str << ": image_gradients : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
// Find contours
std::cout<<"image find contours:"<<std::endl;
std::cout << "image find contours:" << std::endl;
image_find_contours(frame_crop, frame_nbr, i);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " TIME "<<buf_str<<": image_find_contours : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " TIME " << buf_str << ": image_find_contours : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
//saliency map
std::cout<<"image saliency:"<<std::endl;
std::cout << "image saliency:" << std::endl;
image_saliency(frame_crop, frame_nbr, i);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " TIME "<<buf_str<<": image_saliency : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " TIME " << buf_str << ": image_saliency : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
//frame disparity
if(!mode && frame_nbr!=0)
if (!mode && frame_nbr != 0)
{
std::cout<<"frame disparity:"<<std::endl;
std::cout << "frame disparity:" << std::endl;
frame_disparity(pre_frame, frame_crop, frame_nbr, i, 0);
end_t_segmentation = std::chrono::steady_clock::now();
std::cout << " TIME "<<buf_str<<": frame_disparity : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
std::cout << " TIME " << buf_str << ": frame_disparity : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
step_t_segmentation = end_t_segmentation;
}
}
src/class_src/calibration.cpp
+108
View File
@@ -0,0 +1,108 @@
#include "calibration.h"
void readTiff(char *filename, double *adfGeoTransform)
{
GDALDataset *poDataset;
GDALAllRegister();
poDataset = (GDALDataset *)GDALOpen(filename, GA_ReadOnly);
if (poDataset != NULL)
{
poDataset->GetGeoTransform(adfGeoTransform);
}
}
void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff)
{
YAML::Node config = YAML::LoadFile(cameraCalib);
const YAML::Node &node_test1 = config["camera_matrix"];
float data_cm[9];
for (std::size_t i = 0; i < node_test1["data"].size(); i++)
data_cm[i] = node_test1["data"][i].as<float>();
cv::Mat cameraMat_ = cv::Mat(3, 3, CV_32F, data_cm);
cameraMat = cameraMat_.clone();
std::cout << cameraMat << std::endl;
const YAML::Node &node_test2 = config["distortion_coefficients"];
float data_dc[5];
for (std::size_t i = 0; i < node_test2["data"].size(); i++)
data_dc[i] = node_test2["data"][i].as<float>();
cv::Mat distCoeff_ = cv::Mat(5, 1, CV_32F, data_dc);
distCoeff = distCoeff_.clone();
std::cout << distCoeff << std::endl;
}
void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform)
{
//Returns global coordinates from pixel x, y coordinates
double xoff, a, b, yoff, d, e;
xoff = adfGeoTransform[0];
a = adfGeoTransform[1];
b = adfGeoTransform[2];
yoff = adfGeoTransform[3];
d = adfGeoTransform[4];
e = adfGeoTransform[5];
//printf("%f %f %f %f %f %f\n",xoff, a, b, yoff, d, e );
lon = a * x + b * y + xoff;
lat = d * x + e * y + yoff;
}
void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform)
{
x = int(round((lon - adfGeoTransform[0]) / adfGeoTransform[1]));
y = int(round((lat - adfGeoTransform[3]) / adfGeoTransform[5]));
}
void fillMatrix(cv::Mat &H, double *matrix, bool show)
{
double *vals = (double *)H.data;
for (int i = 0; i < 9; i++)
{
vals[i] = matrix[i];
}
if (show)
std::cout << H << "\n";
}
void read_projection_matrix(cv::Mat &H, char *path)
{
FILE *fp;
char *line = NULL;
size_t len = 0;
ssize_t read;
double proj_matrix[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
int i = 0;
fp = fopen(path, "r");
if (fp == NULL)
exit(EXIT_FAILURE);
while ((read = getline(&line, &len, fp)) != -1)
{
std::cout << line << std::endl;
std::stringstream ss(line);
while (ss >> proj_matrix[i])
i++;
}
fclose(fp);
fillMatrix(H, proj_matrix);
free(line);
}
void convert_coords(std::vector<ObjCoords> &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform)
{
double latitude, longitude;
std::vector<cv::Point2f> x_y, ll;
x_y.push_back(cv::Point2f(x, y));
//transform camera pixel to map pixel
cv::perspectiveTransform(x_y, ll, H);
//tranform to map pixel to map gps
pixel2coord(ll[0].x, ll[0].y, latitude, longitude, adfGeoTransform);
ObjCoords coord;
coord.lat_ = latitude;
coord.long_ = longitude;
coord.class_ = detected_class;
coords.push_back(coord);
}
src/class_src/message.cpp
+109
View File
@@ -0,0 +1,109 @@
#include "message.h"
#include "calibration.h"
unsigned long long time_in_ms()
{
struct timeval tv;
gettimeofday(&tv, NULL);
unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000;
return t_stamp_ms;
}
void addRoadUserfromTracker(const std::vector<Tracker> &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat &maskOrient, double *adfGeoTransform, cv::Mat H)
{
m->t_stamp_ms = time_in_ms();
m->objects.clear();
double lat, lon, alt;
for (auto t : trackers)
{
if (t.pred_list_.size() > 0)
{
Categories cat;
switch (t.class_)
{
case 0:
cat = Categories::C_person;
break;
case 1:
cat = Categories::C_car;
break;
case 2:
cat = Categories::C_car;
break;
case 3:
cat = Categories::C_bus;
break;
case 4:
cat = Categories::C_motorbike;
break;
case 5:
cat = Categories::C_bycicle;
break;
}
//std::cout << t.pred_list_.size() << std::endl;
gc.enu2Geodetic(t.pred_list_.back().x_, t.pred_list_.back().y_, 0, &lat, &lon, &alt);
int pix_x, pix_y;
coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
// TODO: test correctness - added perspective transform call to converter pix_x and pix_y
// sometimes some values are wrong. float ok?
// std::vector<cv::Point2f> map_p, camera_p;
// std::cout<<"--- pix_x, pix_y: "<<pix_x<<", "<<pix_y<<std::endl;
// map_p.push_back(cv::Point2f(pix_x, pix_y));
// std::cout<<"map_p: "<<map_p<<std::endl;
// //transform camera pixel to map pixel
// cv::perspectiveTransform(map_p, camera_p, H.inv());
// std::cout<<"size H: "<<H.cols<<", "<<H.rows<<std::endl;
// std::cout<<"camera_p: "<<camera_p<<std::endl;
// // TODO: in some cases these lines causes seg fault!
// std::cout<<"y, x :"<<camera_p[0].y<<", "<<camera_p[0].x<<std::endl;
// std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
// // std::cout<<"vec3b: "<<(cv::Vec3b)(pix_y,pix_x);
// assert (camera_p[0].x < maskOrient.cols);
// assert (camera_p[0].y < maskOrient.rows);
// uint8_t maskOrientPixel = maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)[0];
// std::cout<<"boo: "<<maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)<<std::endl;
// uint8_t orientation;
// if(maskOrientPixel != 0)
// {
// orientation = maskOrientPixel;
// // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
// }
// else
// {
// orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
// //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
// }
// TODO: to validate -> it works for grayscale image (see demo.cpp, row: "cv::Mat maskOrient = cv::imread(camera->maskFileOrient, 0);")
// TODO: include perspective transform
// std::cout<<"y, x :"<<pix_y<<", "<<pix_x<<std::endl;
// std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
// std::cout<<"point: "<<(cv::Point)(pix_y,pix_x);
// uint8_t maskOrientPixel = maskOrient.at<uchar>(pix_y,pix_x);
// uint8_t orientation;
// if(maskOrientPixel != 0)
// {
// orientation = maskOrientPixel;
// // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
// }
// else
// {
// orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
// //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
// }
uint8_t orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
// std::cout<<"orient: "<<unsigned(orientation)<<std::endl;
//std::cout << "lat: " << lat << " lon: " << lon << std::endl;
uint8_t velocity = uint8_t(std::abs(t.pred_list_.back().vel_ * 3.6 / 2));
// std::cout<<"vel: "<<unsigned(velocity)<<std::endl;
RoadUser r{static_cast<float>(lat), static_cast<float>(lon), velocity, orientation, cat};
//std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << int(r.speed) << " " << int(r.orientation) << " " << r.category << std::endl;
m->objects.push_back(r);
}
}
m->num_objects = m->objects.size();
}
src/class_src/visualization.cpp
+377
View File
@@ -0,0 +1,377 @@
#include "visualization.h"
/* Thread function to show the updated images
**/
void *show_updates(void *x_void_ptr)
{
cv::namedWindow("original", cv::WINDOW_NORMAL);
cv::namedWindow("detection", cv::WINDOW_NORMAL);
cv::namedWindow("topview", cv::WINDOW_NORMAL);
cv::namedWindow("disparity", cv::WINDOW_NORMAL);
cv::Mat original_loc, detection_loc, topview_loc, disparity_loc;
bool update_o_loc, update_de_loc, update_t_loc, update_di_loc;
while (gRun)
{
TIMER_START
// critical section: copy the struct in local variable
// in this way we can unlock the sem for the main thread
if (updates.mutex_o.try_lock())
{
update_o_loc = updates.update_o;
updates.update_o = false;
if (update_o_loc)
original_loc = updates.original.clone();
updates.mutex_o.unlock();
}
if (updates.mutex_de.try_lock())
{
update_de_loc = updates.update_de;
updates.update_de = false;
if (update_de_loc)
detection_loc = updates.detection.clone();
updates.mutex_de.unlock();
}
if (updates.mutex_t.try_lock())
{
update_t_loc = updates.update_t;
updates.update_t = false;
if (update_t_loc)
topview_loc = updates.topview.clone();
updates.mutex_t.unlock();
}
if (updates.mutex_di.try_lock())
{
update_di_loc = updates.update_di;
updates.update_di = false;
if (update_di_loc)
disparity_loc = updates.disparity.clone();
updates.mutex_di.unlock();
}
if (update_o_loc)
cv::imshow("original", original_loc);
if (update_de_loc)
cv::imshow("detection", detection_loc);
if (update_t_loc)
cv::imshow("topview", topview_loc);
if (update_di_loc)
cv::imshow("disparity", disparity_loc);
cv::waitKey(1);
// usleep(20000); //sleep 20 msec
std::cout << "show_updates: ";
TIMER_STOP
}
return (void *)0;
}
void *originalFrame(void *x_void_ptr)
{
Frame_t *info_show_orig = (Frame_t *)x_void_ptr;
cv::Mat frame_loc;
int frame_nbr_loc = 0;
while (gRun)
{
TIMER_START
// critical section: copy the struct in local variable
// in this way we can unlock the sem for the main thread
info_show_orig->sem_vc.lock();
frame_loc = info_show_orig->frame.clone();
frame_nbr_loc = info_show_orig->frame_nbr;
info_show_orig->sem_vc.unlock();
if (frame_nbr_loc == 0)
{
usleep(1000000);
printf("no frame received\n");
continue;
}
updates.mutex_o.lock();
updates.original = frame_loc.clone();
updates.update_o = true;
updates.mutex_o.unlock();
usleep(10000); //sleep 10 msec
std::cout << "originalFrame: ";
TIMER_STOP
}
return (void *)0;
}
void *detectionFrame(void *x_void_ptr)
{
ModFrame_t *info_show = (ModFrame_t *)x_void_ptr;
double lat, lon, alt;
int pix_x, pix_y;
cv::Mat original_frame_loc;
std::vector<Tracker> trackers;
geodetic_converter::GeodeticConverter gc;
double adfGeoTransform[6];
cv::Mat H;
tk::dnn::Yolo3Detection yolo;
int num_detected;
cv::Mat mask;
// box variable
tk::dnn::box b;
int x0, w, x1, y0, h, y1;
int objClass;
std::string det_class;
;
// float prob;
cv::Scalar intensity;
std::vector<cv::Point2f> map_p, camera_p;
int baseline = 0;
float fontScale = 0.5;
int thickness = 2;
while (gRun)
{
TIMER_START
// critical section: copy the struct in local variable
// in this way we can unlock the sem for the main thread
info_show->sem.lock();
original_frame_loc = info_show->original_frame.clone();
// std::vector<Tracker> trackers;
trackers = info_show->trackers;
// geodetic_converter::GeodeticConverter gc;
gc = info_show->gc;
for (int i = 0; i < 6; i++)
adfGeoTransform[i] = info_show->adfGeoTransform[i];
// cv::Mat H;
H = info_show->H.clone();
yolo = info_show->yolo;
mask = info_show->mask.clone();
info_show->sem.unlock();
if (trackers.empty())
{
usleep(1000000);
printf("no data available\n");
continue;
}
num_detected = yolo.detected.size();
for (int i = 0; i < num_detected; i++)
{
b = yolo.detected[i];
x0 = b.x;
w = b.w;
x1 = b.x + w;
y0 = b.y;
h = b.h;
y1 = b.y + h;
objClass = b.cl;
det_class = obj_class[b.cl];
// prob = b.prob;
intensity = mask.at<uchar>(cv::Point(int(x0 + b.w / 2), y1));
if (intensity[0] && objClass < 6)
{
//std::cout<<objClass<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
cv::rectangle(original_frame_loc, cv::Point(x0, y0), cv::Point(x1, y1), yolo.colors[objClass], 2);
// draw label
cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline);
cv::rectangle(original_frame_loc, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), yolo.colors[b.cl], -1);
cv::putText(original_frame_loc, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness);
}
}
for (auto t : trackers)
{
for (size_t p = 1; p < t.pred_list_.size(); p++)
{
gc.enu2Geodetic(t.pred_list_[p].x_, t.pred_list_[p].y_, 0, &lat, &lon, &alt);
coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
map_p.clear();
camera_p.clear();
map_p.push_back(cv::Point2f(pix_x, pix_y));
//transform camera pixel to map pixel
cv::perspectiveTransform(map_p, camera_p, H.inv());
// std::cout<<"x,y: "<<pix_x<<", "<<pix_y<<std::endl;
// std::cout<<"map_p: "<<map_p<<std::endl;
// std::cout<<"camera_p: "<<camera_p<<std::endl;
// std::cout<<"size original_frame_loc: "<<original_frame_loc.cols<<", "<<original_frame_loc.rows<<std::endl;
// assert (camera_p[0].x < original_frame_loc.cols);
// assert (camera_p[0].y < original_frame_loc.rows);
if (camera_p[0].x < original_frame_loc.cols && camera_p[0].y < original_frame_loc.rows && camera_p[0].x >= 0 && camera_p[0].y >= 0)
cv::circle(original_frame_loc, cv::Point(camera_p[0].x, camera_p[0].y), 3.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0);
}
}
updates.mutex_de.lock();
updates.detection = original_frame_loc.clone();
updates.update_de = true;
updates.mutex_de.unlock();
std::cout << "detectionFrame: ";
TIMER_STOP
}
return (void *)0;
}
void *topviewFrame(void *x_void_ptr)
{
ModFrame_t *info_show = (ModFrame_t *)x_void_ptr;
double lat, lon, alt;
int pix_x, pix_y;
cv::Mat frame_top;
cv::Mat original_frame_top;
// original_frame_top = cv::imread("../demo/demo/data/map/map_geo.jpg");
original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670.png");
// original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670_V.png");
std::vector<Tracker> trackers;
geodetic_converter::GeodeticConverter gc;
double adfGeoTransform[6];
cv::Mat H;
while (gRun)
{
TIMER_START
// critical section: copy the struct in local variable
// in this way we can unlock the sem for the main thread
info_show->sem.lock();
// std::vector<Tracker> trackers;
trackers = info_show->trackers;
// geodetic_converter::GeodeticConverter gc;
gc = info_show->gc;
for (int i = 0; i < 6; i++)
adfGeoTransform[i] = info_show->adfGeoTransform[i];
// cv::Mat H;
H = info_show->H.clone();
info_show->sem.unlock();
if (trackers.empty())
{
usleep(1000000);
printf("no data available\n");
continue;
}
frame_top = original_frame_top.clone();
for (auto t : trackers)
{
for (size_t p = 1; p < t.pred_list_.size(); p++)
{
gc.enu2Geodetic(t.pred_list_[p].x_, t.pred_list_[p].y_, 0, &lat, &lon, &alt);
coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
if (pix_x < frame_top.cols && pix_y < frame_top.rows && pix_x >= 0 && pix_y >= 0)
cv::circle(frame_top, cv::Point(pix_x, pix_y), 7.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0);
}
}
//outputVideo<< frame_top;
// ------------------------------------------------
updates.mutex_t.lock();
updates.topview = frame_top.clone();
updates.update_t = true;
updates.mutex_t.unlock();
std::cout << "topviewFrame: ";
TIMER_STOP
}
return (void *)0;
}
void *disparityFrame(void *x_void_ptr)
{
Frame_t *info_show_disparity = (Frame_t *)x_void_ptr;
bool first_iteration = true;
cv::Mat frame_loc;
int frame_nbr_loc = 0, pre_frame_nbr_loc = 0;
auto start_t = std::chrono::steady_clock::now();
auto step_t = std::chrono::steady_clock::now();
auto end_t = std::chrono::steady_clock::now();
// information for the disparity map
cv::Mat canny, pre_canny, canny_RGB, pre_canny_RGB;
cv::Mat canny_img;
cv::Mat disparity_frame;
while (gRun)
{
start_t = std::chrono::steady_clock::now();
step_t = start_t;
// critical section: copy the struct in local variable
// in this way we can unlock the sem for the main thread
info_show_disparity->sem_vc.lock();
frame_loc = info_show_disparity->frame.clone();
frame_nbr_loc = info_show_disparity->frame_nbr;
info_show_disparity->sem_vc.unlock();
if (frame_nbr_loc == 0)
{
usleep(1000000);
printf("no frame received\n");
continue;
}
// compute frame disparity only in there is a new frame
if (frame_nbr_loc - pre_frame_nbr_loc > 0)
{
pre_frame_nbr_loc = frame_nbr_loc;
//preprocessing frame
step_t = std::chrono::steady_clock::now();
// src_gray
canny_img = img_laplacian(frame_loc, 0);
cv::Canny(canny_img, canny, 100, 100 * 2);
// sprintf(buf_frame_crop_name,"../demo/demo/data/img_disparity/%d_%d_canny.jpg",frame_nbr_loc, 999);
// cv::imwrite(buf_frame_crop_name, canny);
end_t = std::chrono::steady_clock::now();
std::cout << " TIME END pre canny : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// std::cout<<"o: "<<frame_loc.cols<<" - "<<frame_loc.rows<<std::endl;
// std::cout<<"canny: "<<canny.cols<<" - "<<canny.rows<<std::endl;
// std::cout<<"pre: "<<pre_canny.cols<<" - "<<pre_canny.rows<<std::endl;
if (!first_iteration)
{
// backtorgb = cv::cvtColor(pre_canny,cv::COLOR_GRAY2RGB)
cv::cvtColor(pre_canny, pre_canny_RGB, CV_GRAY2RGB);
cv::cvtColor(canny, canny_RGB, CV_GRAY2RGB);
disparity_frame = frame_disparity(pre_canny_RGB, canny_RGB, frame_nbr_loc, 999, 0);
// std::cout<<"size: "<<disparity_frame.rows<<" - "<<disparity_frame.cols<<std::endl;
// if (disparity_frame.rows == 0 || disparity_frame.cols == 0)
// return -1;
// if (disparity_frame.empty())
// { // only fools don't check...
// std::cout << "image not loaded !" << std::endl;
// return -1;
// }
end_t = std::chrono::steady_clock::now();
std::cout << " TIME canny : frame_disparity : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// //--------------------------------
// //frame box disparity on the original image
// step_t_segmentation = std::chrono::steady_clock::now();
// frame_box_disparity(pre_frame, frame, pre_rois, frame_nbr_loc);
// // reset pre_rois for the new roi of the current frame
// // pre_rois.erase(pre_rois.begin(), pre_rois.end());
// end_t_segmentation = std::chrono::steady_clock::now();
// std::cout << " TIME Frame disparity : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
// step_t_segmentation = end_t_segmentation;
// //frame box disparity on the preprocessed image
// cv::cvtColor(pre_canny, pre_canny_RGB, CV_GRAY2RGB);
// cv::cvtColor(canny, canny_RGB, CV_GRAY2RGB);
// frame_box_disparity(pre_canny_RGB, canny_RGB, pre_rois, frame_nbr_loc);
// // reset pre_rois for the new roi of the current frame
// pre_rois.erase(pre_rois.begin(), pre_rois.end());
// end_t_segmentation = std::chrono::steady_clock::now();
// std::cout << " TIME Canny Frame disparity : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
// step_t_segmentation = end_t_segmentation;
// //---------------------------------
updates.mutex_di.lock();
updates.disparity = disparity_frame.clone();
updates.update_di = true;
updates.mutex_di.unlock();
}
pre_canny = canny.clone();
if (first_iteration)
first_iteration = false;
end_t = std::chrono::steady_clock::now();
std::cout << "disparityFrame : TIME END pre canny : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - start_t).count() << " ms" << std::endl;
}
}
return (void *)0;
}