mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

merge

Browse Source
This commit is contained in:
Francesco Gatti
2020-06-01 15:34:59 +02:00
parent 18794d52c2 0548a662f5
commit d4e0d07e09
20 changed files with 1466 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6 -1
View File
@@ -123,13 +123,16 @@ rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files
```
In general the demo program takes 4 parameters:
```
./demo <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes>
./demo <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes> <n-batches> <show-flag>
```
where
* ```<network-rt-file>``` is the rt file generated by a test
* ```<<path-to-video>``` is the path to a video file or a camera input
* ```<kind-of-network>``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
* ```<number-of-classes>```is the number of classes the network is trained on
* ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
* ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
N.b. By default it is used FP32 inference
![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif)
@@ -218,6 +221,8 @@ cd build
./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml
```
This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to subit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate).
## Existing tests and supported networks
| Test Name | Network | Dataset | N Classes | Input size | Weights |
demo/demo/demo.cpp
+46 -21
View File
@@ -34,6 +34,18 @@ int main(int argc, char *argv[]) {
int n_classes = 80;
if(argc > 4)
n_classes = atoi(argv[4]);
int n_batch = 1;
if(argc > 5)
n_batch = atoi(argv[5]);
bool show = true;
if(argc > 6)
show = atoi(argv[6]);
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
if(!show)
SAVE_RESULT = true;
tk::dnn::Yolo3Detection yolo;
tk::dnn::CenternetDetection cnet;
@@ -57,7 +69,7 @@ int main(int argc, char *argv[]) {
FatalError("Network type not allowed (3rd parameter)\n");
}
detNN->init(net, n_classes);
detNN->init(net, n_classes, n_batch);
gRun = true;
@@ -75,27 +87,40 @@ int main(int argc, char *argv[]) {
}
cv::Mat frame;
cv::Mat dnn_input;
cv::namedWindow("detection", cv::WINDOW_NORMAL);
std::vector<tk::dnn::box> detected_bbox;
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
std::vector<cv::Mat> batch_frame;
std::vector<cv::Mat> batch_dnn_input;
while(gRun) {
cap >> frame;
if(!frame.data) {
break;
}
// this will be resized to the net format
dnn_input = frame.clone();
batch_dnn_input.clear();
batch_frame.clear();
//inference
detNN->update(dnn_input);
frame = detNN->draw(frame);
for(int bi=0; bi< n_batch; ++bi){
cap >> frame;
if(!frame.data)
break;
batch_frame.push_back(frame);
cv::imshow("detection", frame);
cv::waitKey(1);
if(SAVE_RESULT)
// this will be resized to the net format
batch_dnn_input.push_back(frame.clone());
}
if(!frame.data)
break;
//inference
detNN->update(batch_dnn_input, n_batch);
detNN->draw(batch_frame);
if(show){
for(int bi=0; bi< n_batch; ++bi){
cv::imshow("detection", batch_frame[bi]);
cv::waitKey(1);
}
}
if(n_batch == 1 && SAVE_RESULT)
resultVideo << frame;
}
@@ -103,10 +128,10 @@ int main(int argc, char *argv[]) {
double mean = 0;
std::cout<<COL_GREENB<<"\n\nTime stats:\n";
std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";
for(int i=0; i<detNN->stats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
std::cout<<"Avg: "<<mean/n_batch<<" ms\t"<<1000/(mean/n_batch)<<" FPS\n"<<COL_END;
return 0;
demo/demo/map.cpp
+46 -24
View File
@@ -34,6 +34,7 @@ int main(int argc, char *argv[])
bool show = false;
bool write_dets = false;
bool write_res_on_file = true;
bool write_coco_json = true;
int n_images = 5000;
bool verbose;
@@ -43,6 +44,7 @@ int main(int argc, char *argv[])
double vm_total = 0, rss_total = 0;
double vm, rss;
//read args
if(argc > 1)
net = argv[1];
if(argc > 2)
@@ -52,6 +54,7 @@ int main(int argc, char *argv[])
if(argc > 4)
config_filename = argv[4];
//check if files needed exist
if(!fileExist(config_filename))
FatalError("Wrong config file path.");
if(!fileExist(net))
@@ -63,26 +66,31 @@ int main(int argc, char *argv[])
tk::dnn::readmAPParams( config_filename, classes, map_points, map_levels, map_step,
IoU_thresh, conf_thresh, verbose);
std::ofstream times, memory;
//extract network name from rt path
std::string net_name;
removePathAndExtension(net, net_name);
std::cout<<"Network: "<<net_name<<std::endl;
//open files (if needed)
std::ofstream times, memory, coco_json;
if(write_coco_json){
coco_json.open(net_name+"_COCO_res.json");
coco_json << "[\n";
}
if(write_res_on_file){
times.open("times_"+net_name+".csv");
memory.open("memory.csv", std::ios_base::app);
memory<<net<<";";
}
// instantiate detector
tk::dnn::Yolo3Detection yolo;
tk::dnn::CenternetDetection cnet;
tk::dnn::MobilenetDetection mbnet;
tk::dnn::DetectionNN *detNN;
int n_classes = classes;
switch(ntype){
case 'y':
detNN = &yolo;
@@ -97,9 +105,9 @@ int main(int argc, char *argv[])
default:
FatalError("Network type not allowed (3rd parameter)\n");
}
detNN->init(net, n_classes);
//read images
std::ifstream all_labels(labels_path);
std::string l_filename;
std::vector<tk::dnn::Frame> images;
@@ -124,21 +132,25 @@ int main(int argc, char *argv[])
FatalError("Wrong image file path.");
cv::Mat frame = cv::imread(f.iFilename.c_str(), cv::IMREAD_COLOR);
std::vector<cv::Mat> batch_frames;
batch_frames.push_back(frame);
int height = frame.rows;
int width = frame.cols;
cv::Mat dnn_input;
if(!frame.data)
break;
dnn_input = frame.clone();
std::vector<cv::Mat> batch_dnn_input;
batch_dnn_input.push_back(frame.clone());
//inference
detected_bbox.clear();
detNN->update(dnn_input, write_res_on_file, &times);
frame = detNN->draw(frame);
detNN->update(batch_dnn_input,1,write_res_on_file, &times, write_coco_json);
detNN->draw(batch_frames);
detected_bbox = detNN->detected;
if(write_coco_json)
printJsonCOCOFormat(&coco_json, f.iFilename.c_str(), detected_bbox, classes, width, height);
std::ofstream myfile;
if(write_dets)
myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("000")));
@@ -160,30 +172,33 @@ int main(int argc, char *argv[])
myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n";
if(show)// draw rectangle for detection
cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2);
cv::rectangle(batch_frames[0], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2);
}
if(write_dets)
myfile.close();
// read and save groundtruth labels
std::ifstream labels(l_filename);
for(std::string line; std::getline(labels, line); ){
std::istringstream in(line);
tk::dnn::BoundingBox b;
in >> b.cl >> b.x >> b.y >> b.w >> b.h;
b.prob = 1;
b.truthFlag = 1;
f.gt.push_back(b);
if(fileExist(f.lFilename.c_str()))
{
std::ifstream labels(l_filename);
for(std::string line; std::getline(labels, line); ){
std::istringstream in(line);
tk::dnn::BoundingBox b;
in >> b.cl >> b.x >> b.y >> b.w >> b.h;
b.prob = 1;
b.truthFlag = 1;
f.gt.push_back(b);
if(show)// draw rectangle for groundtruth
cv::rectangle(frame, cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2);
if(show)// draw rectangle for groundtruth
cv::rectangle(batch_frames[0], cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2);
}
}
images.push_back(f);
if(show){
cv::imshow("detection", frame);
cv::imshow("detection", batch_frames[0]);
cv::waitKey(0);
}
@@ -193,6 +208,13 @@ int main(int argc, char *argv[])
}
if(write_coco_json){
coco_json.seekp (coco_json.tellp() - std::streampos(2));
coco_json << "\n]\n";
coco_json.close();
}
std::cout << "Avg VM[MB]: " << vm_total/images_done/1024.0 << ";Avg RSS[MB]: " << rss_total/images_done/1024.0 << std::endl;
//compute mAP
include/tkDNN/CenternetDetection.h
+3 -3
View File
@@ -73,9 +73,9 @@ public:
CenternetDetection() {};
~CenternetDetection() {};
bool init(const std::string& tensor_path, const int n_classes=80);
void preprocess(cv::Mat &frame);
void postprocess();
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
};
include/tkDNN/DetectionNN.h
+53 -34
View File
@@ -14,7 +14,7 @@
#include "tkdnn.h"
//#define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
#ifdef OPENCV_CUDACONTRIB
#include <opencv2/cudawarping.hpp>
@@ -30,10 +30,12 @@ class DetectionNN {
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_d;
cv::Size originalSize;
std::vector<cv::Size> originalSize;
cv::Scalar colors[256];
int nBatches = 1;
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat bgr[3];
cv::cuda::GpuMat imagePreproc;
@@ -47,21 +49,26 @@ class DetectionNN {
* This method preprocess the image, before feeding it to the NN.
*
* @param frame original frame to adapt for inference.
* @param bi batch index
*/
virtual void preprocess(cv::Mat &frame) = 0;
virtual void preprocess(cv::Mat &frame, const int bi=0) = 0;
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param bi batch index
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
virtual void postprocess() = 0;
virtual void postprocess(const int bi=0,const bool mAP=false) = 0;
public:
int classes = 0;
float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
std::vector<tk::dnn::box> detected; /*bounding boxes in output*/
std::vector<std::vector<tk::dnn::box>> batchDetected; /*bounding boxes in output*/
std::vector<double> stats; /*keeps track of inference times (ms)*/
std::vector<std::string> classesNames;
@@ -74,49 +81,60 @@ class DetectionNN {
*
* @param tensor_path path to the rt file og the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference
* @return true if everything is correct, false otherwise.
*/
virtual bool init(const std::string& tensor_path, const int n_classes=80) = 0;
virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1) = 0;
/**
* This method performs the whole detection of the NN.
*
* @param frame frame to run detection on.
* @param frames frames to run detection on.
* @param cur_batches number of batches to use in inference
* @param save_times if set to true, preprocess, inference and postprocess times
* are saved on a csv file, otherwise not.
* @param times pointer to the output stream where to write times
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){
if(!frame.data)
FatalError("No image data feed to detection");
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){
if(save_times && times==nullptr)
FatalError("save_times set to true, but no valid ofstream given");
if(cur_batches > nBatches)
FatalError("A batch size greater than nBatches cannot be used");
originalSize = frame.size();
printCenteredTitle(" TENSORRT detection ", '=', 30);
originalSize.clear();
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
{
TIMER_START
preprocess(frame);
for(int bi=0; bi<cur_batches;++bi){
if(!frames[bi].data)
FatalError("No image data feed to detection");
originalSize.push_back(frames[bi].size());
preprocess(frames[bi], bi);
}
TIMER_STOP
if(save_times) *times<<t_ns<<";";
}
//do inference
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = cur_batches;
{
dim.print();
if(TKDNN_VERBOSE) dim.print();
TIMER_START
netRT->infer(dim, input_d);
TIMER_STOP
dim.print();
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
batchDetected.clear();
{
TIMER_START
postprocess();
for(int bi=0; bi<cur_batches;++bi)
postprocess(bi, mAP);
TIMER_STOP
if(save_times) *times<<t_ns<<"\n";
}
@@ -125,10 +143,9 @@ class DetectionNN {
/**
* Method to draw boundixg boxes and labels on a frame.
*
* @param frame orginal frame to draw bounding box on.
* @return frame with boundig boxes.
* @param frames orginal frame to draw bounding box on.
*/
cv::Mat draw(cv::Mat &frame) {
void draw(std::vector<cv::Mat>& frames) {
tk::dnn::box b;
int x0, w, x1, y0, h, y1;
int objClass;
@@ -137,24 +154,26 @@ class DetectionNN {
int baseline = 0;
float font_scale = 0.5;
int thickness = 2;
// draw dets
for(int i=0; i<detected.size(); i++) {
b = detected[i];
x0 = b.x;
x1 = b.x + b.w;
y0 = b.y;
y1 = b.y + b.h;
det_class = classesNames[b.cl];
// draw rectangle
cv::rectangle(frame, cv::Point(x0, y0), cv::Point(x1, y1), colors[b.cl], 2);
for(int bi=0; bi<frames.size(); ++bi){
// draw dets
for(int i=0; i<batchDetected[bi].size(); i++) {
b = batchDetected[bi][i];
x0 = b.x;
x1 = b.x + b.w;
y0 = b.y;
y1 = b.y + b.h;
det_class = classesNames[b.cl];
// draw label
cv::Size text_size = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
cv::rectangle(frame, cv::Point(x0, y0), cv::Point((x0 + text_size.width - 2), (y0 - text_size.height - 2)), colors[b.cl], -1);
cv::putText(frame, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness);
// draw rectangle
cv::rectangle(frames[bi], cv::Point(x0, y0), cv::Point(x1, y1), colors[b.cl], 2);
// draw label
cv::Size text_size = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
cv::rectangle(frames[bi], cv::Point(x0, y0), cv::Point((x0 + text_size.width - 2), (y0 - text_size.height - 2)), colors[b.cl], -1);
cv::putText(frames[bi], det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness);
}
}
return frame;
}
};
include/tkDNN/Layer.h
+2 -1
View File
@@ -535,6 +535,7 @@ struct box {
int cl;
float x, y, w, h;
float prob;
std::vector<float> probs;
void print()
{
@@ -581,7 +582,7 @@ public:
dnnType *predictions;
static const int MAX_DETECTIONS = 2048;
static const int MAX_DETECTIONS = 8192;
static Yolo::detection *allocateDetections(int nboxes, int classes);
static void mergeDetections(Yolo::detection *dets, int ndets, int classes);
};
include/tkDNN/MobilenetDetection.h
+3 -3
View File
@@ -65,9 +65,9 @@ public:
MobilenetDetection() {};
~MobilenetDetection() {};
bool init(const std::string& tensor_path, const int n_classes);
void preprocess(cv::Mat &frame);
void postprocess();
bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
};
include/tkDNN/Yolo3Detection.h
+3 -3
View File
@@ -24,9 +24,9 @@ public:
Yolo3Detection() {};
~Yolo3Detection() {};
bool init(const std::string& tensor_path, const int n_classes=80);
void preprocess(cv::Mat &frame);
void postprocess();
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
};
include/tkDNN/evaluation.h
+3
View File
@@ -108,6 +108,9 @@ void computeTPFPFN( std::vector<Frame> &images,const int classes,
bool verbose=false, const bool write_on_file=false,
std::string net="");
void printJsonCOCOFormat(std::ofstream *out_file, const std::string image_path, std::vector<tk::dnn::box> bbox, const int classes, const int w, const int h);
}}
#endif /*EVALUATION_H*/
include/tkDNN/utils.h
+5 -3
View File
@@ -36,16 +36,18 @@
#define COL_PURPLEB "\033[1;35m"
#define COL_CYANB "\033[1;36m"
#define TKDNN_VERBOSE 0
// Simple Timer
#define TIMER_START timespec start, end; \
clock_gettime(CLOCK_MONOTONIC, &start);
#define TIMER_STOP_C(col) clock_gettime(CLOCK_MONOTONIC, &end); \
#define TIMER_STOP_C(col, show) clock_gettime(CLOCK_MONOTONIC, &end); \
double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 + \
(double)(end.tv_nsec - start.tv_nsec))/1.0e6; \
std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END;
if(show) std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END;
#define TIMER_STOP TIMER_STOP_C(COL_CYANB)
#define TIMER_STOP TIMER_STOP_C(COL_CYANB, TKDNN_VERBOSE)
/********************************************************
* Prints the error message, and exits
scripts/test_all_tests.sh
+12 -11
View File
@@ -68,28 +68,29 @@ do
export TKDNN_BATCHSIZE=2
echo -e "${ORANGE}Batch $TKDNN_BATCHSIZE ${NC}"
test_net mnist
./test_imuodom &>> $out_file
print_output $? imuodom
test_net yolo4
test_net resnet101_cnet
test_net yolo4_berkeley
test_net yolo3
test_net yolo3_berkeley
test_net yolo3_coco4
test_net yolo3_flir
test_net yolo3_512
test_net yolo3tiny
test_net yolo3tiny_512
test_net yolo2
test_net yolo2_voc
#test_net yolo2tiny
test_net csresnext50-panet-spp
#test_net csresnext50-panet-spp_berkeley
test_net mobilenetv2ssd
test_net yolo3tiny_512
#test_net yolo2tiny
test_net mobilenetv2ssd512
test_net mnist
test_net yolo2
test_net yolo3_berkeley
test_net yolo2_voc
test_net resnet101_cnet
test_net dla34_cnet
test_net yolo3_coco4
test_net mobilenetv2ssd
test_net mobilenetv2ssd512
test_net bdd-mobilenetv2ssd
done
echo "If errors occured, check logfile $out_file"
src/CenternetDetection.cpp
+15 -13
View File
@@ -3,10 +3,11 @@
namespace tk { namespace dnn {
bool CenternetDetection::init(const std::string& tensor_path, const int n_classes){
bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
classes = n_classes;
nBatches = n_batches;
dim = netRT->input_dim;
@@ -41,7 +42,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe
trans = cv::Mat(cv::Size(3,2), CV_32F);
trans2 = cv::Mat(cv::Size(3,2), CV_32F);
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot()));
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
dim_hm = tk::dnn::dataDim_t(1, 80, 128, 128, 1);
dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
@@ -98,7 +99,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe
checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
#else
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot()));
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot()* nBatches));
mean << 0.408, 0.447, 0.47;
stddev << 0.289, 0.274, 0.278;
#endif
@@ -120,13 +121,13 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe
}
void CenternetDetection::preprocess(cv::Mat &frame){
void CenternetDetection::preprocess(cv::Mat &frame, const int bi){
// -----------------------------------pre-process ------------------------------------------
// auto start_t = std::chrono::steady_clock::now();
// auto step_t = std::chrono::steady_clock::now();
// auto end_t = std::chrono::steady_clock::now();
cv::Size sz = originalSize;
cv::Size sz = originalSize[bi];
// std::cout<<"image: "<<sz.width<<", "<<sz.height<<std::endl;
cv::Size sz_old;
float scale = 1.0;
@@ -212,7 +213,7 @@ void CenternetDetection::preprocess(cv::Mat &frame){
// std::cout << " TIME normalize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
checkCuda(cudaMemcpy(input_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
@@ -254,18 +255,18 @@ void CenternetDetection::preprocess(cv::Mat &frame){
int idx = i*imageF.rows*imageF.cols;
int ch = dim2.c-3 +i;
// std::cout<<"i: "<<i<<", idx: "<<idx<<", ch: "<<ch<<std::endl;
memcpy((void*)&input[idx], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
memcpy((void*)&input[idx+ netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
#endif
}
void CenternetDetection::postprocess(){
void CenternetDetection::postprocess(const int bi, const bool mAP){
dnnType *rt_out[4];
rt_out[0] = (dnnType *)netRT->buffersRT[1];
rt_out[1] = (dnnType *)netRT->buffersRT[2];
rt_out[2] = (dnnType *)netRT->buffersRT[3];
rt_out[3] = (dnnType *)netRT->buffersRT[4];
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
// auto start_t = std::chrono::steady_clock::now();
// auto step_t = std::chrono::steady_clock::now();
@@ -389,6 +390,7 @@ void CenternetDetection::postprocess(){
}
}
batchDetected.push_back(detected);
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME detections: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
src/MobilenetDetection.cpp
+18 -12
View File
@@ -126,11 +126,12 @@ float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){
return iou;
}
bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes){
bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str());
imageSize = netRT->input_dim.h;
classes = n_classes;
nBatches = n_batches;
SSDSpec specs[N_SSDSPEC];
@@ -157,9 +158,9 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe
generate_ssd_priors(specs, N_SSDSPEC);
#ifndef OPENCV_CUDACONTRIB
checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot()));
checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
#endif
checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot()));
checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
locations_h = (float *)malloc(N_COORDS * nPriors * sizeof(float));
confidences_h = (float *)malloc(nPriors * classes * sizeof(float));
@@ -208,7 +209,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe
return 1;
}
void MobilenetDetection::preprocess(cv::Mat &frame){
void MobilenetDetection::preprocess(cv::Mat &frame, const int bi){
#ifdef OPENCV_CUDACONTRIB
//move original image on GPU
cv::cuda::GpuMat orig_img, frame_nomean;
@@ -224,7 +225,7 @@ void MobilenetDetection::preprocess(cv::Mat &frame){
for(int i=0; i < netRT->input_dim.c; i++){
int idx = i * imagePreproc.rows * imagePreproc.cols;
checkCuda( cudaMemcpy((void *)&input_d[idx], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols* sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy((void *)&input_d[idx + netRT->input_dim.tot()*bi], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols* sizeof(float), cudaMemcpyDeviceToDevice) );
}
#else
//resize image, remove mean, divide by std
@@ -237,17 +238,17 @@ void MobilenetDetection::preprocess(cv::Mat &frame){
cv::split(imagePreproc, bgr);
for (int i = 0; i < netRT->input_dim.c; i++){
int idx = i * imagePreproc.rows * imagePreproc.cols;
memcpy((void *)&input[idx], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType));
memcpy((void *)&input[idx + netRT->input_dim.tot()*bi], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d, input, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
#endif
}
void MobilenetDetection::postprocess(){
void MobilenetDetection::postprocess(const int bi, const bool mAP){
//get confidences and locations_h
dnnType *rt_out[2];
rt_out[0] = (dnnType *)netRT->buffersRT[3];
rt_out[1] = (dnnType *)netRT->buffersRT[4];
rt_out[0] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
rt_out[1] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
detected.clear();
@@ -255,8 +256,8 @@ void MobilenetDetection::postprocess(){
checkCuda(cudaMemcpy(locations_h, rt_out[1], N_COORDS * nPriors * sizeof(float), cudaMemcpyDeviceToHost));
convert_locatios_to_boxes_and_center();
int width = originalSize.width;
int height = originalSize.height;
int width = originalSize[bi].width;
int height = originalSize[bi].height;
float *conf_per_class;
for (int i = 1; i < classes; i++){
@@ -273,6 +274,10 @@ void MobilenetDetection::postprocess(){
b.w = locations_h[j * N_COORDS + 2];
b.h = locations_h[j * N_COORDS + 3];
if(mAP)
for(int c=1; c<classes; c++)
b.probs.push_back(confidences_h[c * nPriors + j]);
boxes.push_back(b);
}
}
@@ -298,6 +303,7 @@ void MobilenetDetection::postprocess(){
boxes = remaining;
}
}
batchDetected.push_back(detected);
}
src/Yolo3Detection.cpp
+22 -14
View File
@@ -3,12 +3,16 @@
namespace tk { namespace dnn {
bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes) {
bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches) {
//convert network to tensorRT
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
nBatches = n_batches;
tk::dnn::dataDim_t idim = netRT->input_dim;
idim.n = nBatches;
if(netRT->pluginFactory->n_yolos < 2 ) {
FatalError("this is not yolo3");
}
@@ -19,7 +23,7 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes) {
num = yRT->num;
nMasks = yRT->n_masks;
// make a yolo layer for interpret predictions
// make a yolo layer to interpret predictions
yolo[i] = new tk::dnn::Yolo(nullptr, classes, nMasks, ""); // yolo without input and bias
yolo[i]->mask_h = new dnnType[nMasks];
yolo[i]->bias_h = new dnnType[num*nMasks*2];
@@ -31,9 +35,9 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes) {
dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
#ifndef OPENCV_CUDACONTRIB
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot()));
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*idim.tot()));
#endif
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot()));
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*idim.tot()));
// class colors precompute
for(int c=0; c<classes; c++) {
@@ -48,7 +52,7 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes) {
return true;
}
void Yolo3Detection::preprocess(cv::Mat &frame){
void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat orig_img, img_resized;
orig_img = cv::cuda::GpuMat(frame);
@@ -64,7 +68,7 @@ void Yolo3Detection::preprocess(cv::Mat &frame){
int size = imagePreproc.rows * imagePreproc.cols;
int ch = netRT->input_dim.c-1 -i;
bgr[ch].download(bgr_h); //TODO: don't copy back on CPU
checkCuda( cudaMemcpy(input_d + i*size, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice));
checkCuda( cudaMemcpy(input_d + i*size + netRT->input_dim.tot()*bi, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice));
}
#else
cv::resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
@@ -77,21 +81,21 @@ void Yolo3Detection::preprocess(cv::Mat &frame){
for(int i=0; i<netRT->input_dim.c; i++) {
int idx = i*imagePreproc.rows*imagePreproc.cols;
int ch = netRT->input_dim.c-1 -i;
memcpy((void*)&input[idx], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType));
memcpy((void*)&input[idx + netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d, input, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
checkCuda(cudaMemcpyAsync(input_d + netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
#endif
}
void Yolo3Detection::postprocess(){
void Yolo3Detection::postprocess(const int bi, const bool mAP){
//get yolo outputs
dnnType *rt_out[netRT->pluginFactory->n_yolos];
for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
rt_out[i] = (dnnType*)netRT->buffersRT[i+1];
}
for(int i=0; i<netRT->pluginFactory->n_yolos; i++)
rt_out[i] = (dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi;
float x_ratio = float(originalSize.width) / float(netRT->input_dim.w);
float y_ratio = float(originalSize.height) / float(netRT->input_dim.h);
float x_ratio = float(originalSize[bi].width) / float(netRT->input_dim.w);
float y_ratio = float(originalSize[bi].height) / float(netRT->input_dim.h);
// compute dets
nDets = 0;
@@ -132,9 +136,13 @@ void Yolo3Detection::postprocess(){
res.y = y0;
res.w = x1 - x0;
res.h = y1 - y0;
if(mAP)
for(int c=0; c<classes; c++)
res.probs.push_back(dets[j].prob[c]);
detected.push_back(res);
}
}
batchDetected.push_back(detected);
}
src/evaluation.cpp
+42 -1
View File
@@ -314,5 +314,46 @@ void computeTPFPFN( std::vector<Frame> &images,const int classes,
std::cout<<"avg precision: "<<avg_precision<<"\tavg recall: "<<avg_recall<<"\tavg f1 score:"<<f1_score<<std::endl;
}
void printJsonCOCOFormat(std::ofstream *out_file, const std::string image_path, std::vector<tk::dnn::box> bbox, const int classes, const int w, const int h)
{
int coco_ids[] = { 1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,18,19,20,21,22,23,24,25,27,28,31,32,33,34,35,36,37,38,39,40,41,42,43,44,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,67,70,72,73,74,75,76,77,78,79,80,81,82,84,85,86,87,88,89,90 };
std::string id = image_path.substr(image_path.find("images/")+7, image_path.find(".jpg") - image_path.find("images/") -7);
int image_id = std::stoi(id);
for (int i = 0; i < bbox.size(); ++i) {
float xmin = bbox[i].x ;
float xmax = bbox[i].x + float(bbox[i].w);
float ymin = bbox[i].y;
float ymax = bbox[i].y + float(bbox[i].h);
//limit to image borders
if (xmin < 0) xmin = 0;
if (ymin < 0) ymin = 0;
if (xmax > w) xmax = w;
if (ymax > h) ymax = h;
float bx = xmin;
float by = ymin;
float bw = xmax - xmin;
float bh = ymax - ymin;
if(bbox[i].probs.size() == classes)
for (int j = 0; j < classes; ++j) {
//min threshold confidence is set in DetectionNN.h
if (bbox[i].probs[j] > 0) {
*out_file << "{\"image_id\":" << image_id <<
", \"category_id\":" << coco_ids[j] <<
", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh <<
"], \"score\":" << bbox[i].probs[j] << "},\n";
}
}
else
*out_file << "{\"image_id\":" << image_id <<
", \"category_id\":" << coco_ids[bbox[i].cl] <<
", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh <<
"], \"score\":" << bbox[i].prob << "},\n";
}
}
}}
src/kernels/activation.mish.cu → src/kernels/activation_mish.cu
+22 -3
View File
@@ -3,20 +3,39 @@
#define MISH_THRESHOLD 20
__device__ float tanh_activate_kernel(float x){return (2/(1 + expf(-2*x)) - 1);}
__device__ float softplus_kernel(float x, float threshold = 20) {
__device__
float tanh_activate_kernel(float x){return (2/(1 + expf(-2*x)) - 1);}
__device__
float softplus_kernel(float x, float threshold = 20) {
if (x > threshold) return x; // too large
else if (x < -threshold) return expf(x); // too small
return logf(expf(x) + 1);
}
__device__
float mish_yashas(float x) {
float e = __expf(x);
if (x <= -18.0f)
return x * e;
float n = e * e + 2 * e;
if (x <= -5.0f)
return x * __fdividef(n, n + 2);
return x - 2 * __fdividef(x, n + 2);
}
// https://github.com/digantamisra98/Mish
// https://github.com/AlexeyAB/darknet/blob/master/src/activation_kernels.cu
__global__
void activation_mish(dnnType *input, dnnType *output, int size) {
int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
if (i < size)
output[i] = input[i] * tanh_activate_kernel( softplus_kernel(input[i], MISH_THRESHOLD));
// output[i] = input[i] * tanh_activate_kernel( softplus_kernel(input[i], MISH_THRESHOLD));
output[i] = mish_yashas(input[i]);
}
/**
tests/darknet/cfg/yolo4_berkeley.cfg
+1159
View File
File diff suppressed because it is too large Load Diff
tests/darknet/yolo4_berkeley.cpp
+1 -1
View File
@@ -17,7 +17,7 @@ int main() {
std::string wgs_path = bin_path + "/layers";
std::string cfg_path = "../tests/darknet/cfg/yolo4_berkeley.cfg";
std::string name_path = "../tests/darknet/names/berkeley.names";
downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s//download");
downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download");
// parse darknet network
tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
tests/mobilenet/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp
+1 -1
View File
@@ -134,7 +134,7 @@ const char *regression_header5 = "bdd-mobilenetv2ssd/layers/regression_headers-5
int main()
{
// downloadWeightsifDoNotExist(input_bin, "bdd-mobilenetv2ssd", "https://cloud.hipert.unimore.it/s//download");
downloadWeightsifDoNotExist(input_bin, "bdd-mobilenetv2ssd", "https://cloud.hipert.unimore.it/s/jzRBxcEJYJ99RLa/download");
int classes = 11;
tests/test_rtinference/rtinference.cpp
+4 -2
View File
@@ -30,7 +30,8 @@ int main(int argc, char *argv[]) {
int ret_tensorrt = 0;
std::cout<<"Testing with batchsize: "<<BATCH_SIZE<<"\n";
printCenteredTitle(" TENSORRT inference ", '=', 30);
for(int i=0; i<10; i++) {
float total_time = 0;
for(int i=0; i<1200; i++) {
// generate input
for(int j=0; j<netRT.input_dim.tot(); j++) {
@@ -44,6 +45,7 @@ int main(int argc, char *argv[]) {
TIMER_START
netRT.infer(dim, input_d);
TIMER_STOP
total_time+= t_ns;
// control output
std::cout<<"Output Buffers: "<<netRT.getBuffersN()-1<<"\n";
@@ -56,6 +58,6 @@ int main(int argc, char *argv[]) {
}
}
}
std::cout<<"avg: "<<total_time/1200.<<std::endl;
return ret_tensorrt;
}