From 533bb4878914982a72b3f0165cc94f4070b8a894 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 11 May 2020 11:57:58 +0200 Subject: [PATCH 001/228] Add json detection creation for codalab check Signed-off-by: Micaela Verucchi --- CMakeLists.txt | 12 + demo/demo/map.cpp | 57 +- include/tkDNN/CenternetDetection.h | 2 +- include/tkDNN/DetectionNN.h | 10 +- include/tkDNN/Layer.h | 3 +- include/tkDNN/MobilenetDetection.h | 2 +- include/tkDNN/Yolo3Detection.h | 2 +- include/tkDNN/evaluation.h | 3 + src/CenternetDetection.cpp | 2 +- src/MobilenetDetection.cpp | 6 +- src/Yolo3Detection.cpp | 5 +- src/evaluation.cpp | 43 +- .../bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp | 2 +- tests/test_rtinference/rtinference.cpp | 6 +- tests/yolo4/yolo4_320.cpp | 666 ++++++++++++++++++ tests/yolo4/yolo4_512.cpp | 666 ++++++++++++++++++ tests/yolo4/yolo4_608.cpp | 666 ++++++++++++++++++ 17 files changed, 2119 insertions(+), 34 deletions(-) create mode 100644 tests/yolo4/yolo4_320.cpp create mode 100644 tests/yolo4/yolo4_512.cpp create mode 100644 tests/yolo4/yolo4_608.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 028d375..7222825 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,18 @@ target_link_libraries(test_yolo3_flir tkDNN) add_executable(test_yolo4 tests/yolo4/yolo4.cpp) target_link_libraries(test_yolo4 tkDNN) +add_executable(test_yolo4_320 tests/yolo4/yolo4_320.cpp) +target_link_libraries(test_yolo4_320 tkDNN) + +add_executable(test_yolo4_512 tests/yolo4/yolo4_512.cpp) +target_link_libraries(test_yolo4_512 tkDNN) + +add_executable(test_yolo4_608 tests/yolo4/yolo4_608.cpp) +target_link_libraries(test_yolo4_608 tkDNN) + +add_executable(test_yolo4_berkeley tests/yolo4_berkeley/yolo4_berkeley.cpp) +target_link_libraries(test_yolo4_berkeley tkDNN) + add_executable(test_mobilenetv2ssd tests/mobilenetv2ssd/mobilenetv2ssd.cpp) target_link_libraries(test_mobilenetv2ssd tkDNN) diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index e12fc36..febb2bc 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -34,6 +34,7 @@ int main(int argc, char *argv[]) bool show = false; bool write_dets = false; bool write_res_on_file = true; + bool write_coco_json = true; int n_images = 5000; bool verbose; @@ -43,6 +44,7 @@ int main(int argc, char *argv[]) double vm_total = 0, rss_total = 0; double vm, rss; + //read args if(argc > 1) net = argv[1]; if(argc > 2) @@ -52,6 +54,7 @@ int main(int argc, char *argv[]) if(argc > 4) config_filename = argv[4]; + //check if files needed exist if(!fileExist(config_filename)) FatalError("Wrong config file path."); if(!fileExist(net)) @@ -63,26 +66,31 @@ int main(int argc, char *argv[]) tk::dnn::readmAPParams( config_filename, classes, map_points, map_levels, map_step, IoU_thresh, conf_thresh, verbose); - std::ofstream times, memory; + //extract network name from rt path std::string net_name; removePathAndExtension(net, net_name); std::cout<<"Network: "<init(net, n_classes); + //read images std::ifstream all_labels(labels_path); std::string l_filename; std::vector images; @@ -135,10 +143,13 @@ int main(int argc, char *argv[]) //inference detected_bbox.clear(); - detNN->update(dnn_input, write_res_on_file, ×); + detNN->update(dnn_input, write_res_on_file, ×, write_coco_json); frame = detNN->draw(frame); detected_bbox = detNN->detected; - + + if(write_coco_json) + printJsonCOCOFormat(&coco_json, f.iFilename.c_str(), detected_bbox, classes, width, height); + std::ofstream myfile; if(write_dets) myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("000"))); @@ -167,17 +178,20 @@ int main(int argc, char *argv[]) myfile.close(); // read and save groundtruth labels - std::ifstream labels(l_filename); - for(std::string line; std::getline(labels, line); ){ - std::istringstream in(line); - tk::dnn::BoundingBox b; - in >> b.cl >> b.x >> b.y >> b.w >> b.h; - b.prob = 1; - b.truthFlag = 1; - f.gt.push_back(b); + if(fileExist(f.lFilename.c_str())) + { + std::ifstream labels(l_filename); + for(std::string line; std::getline(labels, line); ){ + std::istringstream in(line); + tk::dnn::BoundingBox b; + in >> b.cl >> b.x >> b.y >> b.w >> b.h; + b.prob = 1; + b.truthFlag = 1; + f.gt.push_back(b); - if(show)// draw rectangle for groundtruth - cv::rectangle(frame, cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2); + if(show)// draw rectangle for groundtruth + cv::rectangle(frame, cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2); + } } images.push_back(f); @@ -193,6 +207,13 @@ int main(int argc, char *argv[]) } + + if(write_coco_json){ + coco_json.seekp (coco_json.tellp()-2); + coco_json << "\n]\n"; + coco_json.close(); + } + std::cout << "Avg VM[MB]: " << vm_total/images_done/1024.0 << ";Avg RSS[MB]: " << rss_total/images_done/1024.0 << std::endl; //compute mAP diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 10fccec..92feba5 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -75,7 +75,7 @@ public: bool init(const std::string& tensor_path, const int n_classes=80); void preprocess(cv::Mat &frame); - void postprocess(); + void postprocess(const bool mAP=false); }; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index ccb379c..a635b2d 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -14,7 +14,7 @@ #include "tkdnn.h" -//#define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. +#define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. #ifdef OPENCV_CUDACONTRIB #include @@ -55,11 +55,11 @@ class DetectionNN { * boundig boxes. * */ - virtual void postprocess() = 0; + virtual void postprocess(const bool mAP=false) = 0; public: int classes = 0; - float confThreshold = 0.3; /*threshold on the confidence of the boxes*/ + float confThreshold = 0.05; /*threshold on the confidence of the boxes*/ std::vector detected; /*bounding boxes in output*/ std::vector stats; /*keeps track of inference times (ms)*/ @@ -86,7 +86,7 @@ class DetectionNN { * are saved on a csv file, otherwise not. * @param times pointer to the output stream where to write times */ - void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){ + void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ if(!frame.data) FatalError("No image data feed to detection"); @@ -116,7 +116,7 @@ class DetectionNN { { TIMER_START - postprocess(); + postprocess(mAP); TIMER_STOP if(save_times) *times< probs; void print() { @@ -581,7 +582,7 @@ public: dnnType *predictions; - static const int MAX_DETECTIONS = 2048; + static const int MAX_DETECTIONS = 8192; static Yolo::detection *allocateDetections(int nboxes, int classes); static void mergeDetections(Yolo::detection *dets, int ndets, int classes); }; diff --git a/include/tkDNN/MobilenetDetection.h b/include/tkDNN/MobilenetDetection.h index fea1449..7271005 100644 --- a/include/tkDNN/MobilenetDetection.h +++ b/include/tkDNN/MobilenetDetection.h @@ -67,7 +67,7 @@ public: bool init(const std::string& tensor_path, const int n_classes); void preprocess(cv::Mat &frame); - void postprocess(); + void postprocess(const bool mAP=false); }; diff --git a/include/tkDNN/Yolo3Detection.h b/include/tkDNN/Yolo3Detection.h index e8be562..d0acd8b 100644 --- a/include/tkDNN/Yolo3Detection.h +++ b/include/tkDNN/Yolo3Detection.h @@ -26,7 +26,7 @@ public: bool init(const std::string& tensor_path, const int n_classes=80); void preprocess(cv::Mat &frame); - void postprocess(); + void postprocess(const bool mAP=false); }; diff --git a/include/tkDNN/evaluation.h b/include/tkDNN/evaluation.h index d76bb7d..8907d9d 100644 --- a/include/tkDNN/evaluation.h +++ b/include/tkDNN/evaluation.h @@ -108,6 +108,9 @@ void computeTPFPFN( std::vector &images,const int classes, bool verbose=false, const bool write_on_file=false, std::string net=""); + +void printJsonCOCOFormat(std::ofstream *out_file, const std::string image_path, std::vector bbox, const int classes, const int w, const int h); + }} #endif /*EVALUATION_H*/ diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 97668e6..dcdb610 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -260,7 +260,7 @@ void CenternetDetection::preprocess(cv::Mat &frame){ #endif } -void CenternetDetection::postprocess(){ +void CenternetDetection::postprocess(const bool mAP){ dnnType *rt_out[4]; rt_out[0] = (dnnType *)netRT->buffersRT[1]; rt_out[1] = (dnnType *)netRT->buffersRT[2]; diff --git a/src/MobilenetDetection.cpp b/src/MobilenetDetection.cpp index 99b7a0c..d66a8cc 100644 --- a/src/MobilenetDetection.cpp +++ b/src/MobilenetDetection.cpp @@ -243,7 +243,7 @@ void MobilenetDetection::preprocess(cv::Mat &frame){ #endif } -void MobilenetDetection::postprocess(){ +void MobilenetDetection::postprocess(const bool mAP){ //get confidences and locations_h dnnType *rt_out[2]; rt_out[0] = (dnnType *)netRT->buffersRT[3]; @@ -273,6 +273,10 @@ void MobilenetDetection::postprocess(){ b.w = locations_h[j * N_COORDS + 2]; b.h = locations_h[j * N_COORDS + 3]; + if(mAP) + for(int c=1; cpluginFactory->n_yolos]; for(int i=0; ipluginFactory->n_yolos; i++) { @@ -132,6 +132,9 @@ void Yolo3Detection::postprocess(){ res.y = y0; res.w = x1 - x0; res.h = y1 - y0; + if(mAP) + for(int c=0; c &images,const int classes, std::cout<<"avg precision: "< bbox, const int classes, const int w, const int h) +{ + int coco_ids[] = { 1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,18,19,20,21,22,23,24,25,27,28,31,32,33,34,35,36,37,38,39,40,41,42,43,44,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,67,70,72,73,74,75,76,77,78,79,80,81,82,84,85,86,87,88,89,90 }; + std::string id = image_path.substr(image_path.find("images/")+7, image_path.find(".jpg") - image_path.find("images/") -7); + int image_id = std::stoi(id); + for (int i = 0; i < bbox.size(); ++i) { + float xmin = bbox[i].x ; + float xmax = bbox[i].x + float(bbox[i].w); + float ymin = bbox[i].y; + float ymax = bbox[i].y + float(bbox[i].h); + + //limit to image borders + if (xmin < 0) xmin = 0; + if (ymin < 0) ymin = 0; + if (xmax > w) xmax = w; + if (ymax > h) ymax = h; + + float bx = xmin; + float by = ymin; + float bw = xmax - xmin; + float bh = ymax - ymin; + + if(bbox[i].probs.size() == classes) + for (int j = 0; j < classes; ++j) { + //min threshold confidence is set in DetectionNN.h + if (bbox[i].probs[j] > 0) { + + *out_file << "{\"image_id\":" << image_id << + ", \"category_id\":" << coco_ids[j] << + ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << + "], \"score\":" << bbox[i].probs[j] << "},\n"; + } + } + else + *out_file << "{\"image_id\":" << image_id << + ", \"category_id\":" << coco_ids[bbox[i].cl] << + ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << + "], \"score\":" << bbox[i].prob << "},\n"; + } +} + }} diff --git a/tests/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp b/tests/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp index df3e617..1549983 100644 --- a/tests/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp +++ b/tests/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp @@ -134,7 +134,7 @@ const char *regression_header5 = "bdd-mobilenetv2ssd/layers/regression_headers-5 int main() { - // downloadWeightsifDoNotExist(input_bin, "bdd-mobilenetv2ssd", "https://cloud.hipert.unimore.it/s//download"); + downloadWeightsifDoNotExist(input_bin, "bdd-mobilenetv2ssd", "https://cloud.hipert.unimore.it/s/jzRBxcEJYJ99RLa/download"); int classes = 11; diff --git a/tests/test_rtinference/rtinference.cpp b/tests/test_rtinference/rtinference.cpp index 1ca9763..4b6b21f 100644 --- a/tests/test_rtinference/rtinference.cpp +++ b/tests/test_rtinference/rtinference.cpp @@ -30,7 +30,8 @@ int main(int argc, char *argv[]) { int ret_tensorrt = 0; std::cout<<"Testing with batchsize: "<dstData; + + printCenteredTitle(" compute detections ", '=', 30); + TIMER_START + int ndets = 0; + tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); + for (int i = 0; i < 3; i++) + yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); + tk::dnn::Yolo::mergeDetections(dets, ndets, classes); + + for (int j = 0; j < ndets; j++) + { + tk::dnn::Yolo::box b = dets[j].bbox; + int x0 = (b.x - b.w / 2.); + int x1 = (b.x + b.w / 2.); + int y0 = (b.y - b.h / 2.); + int y1 = (b.y + b.h / 2.); + + int cl = 0; + for (int c = 0; c < classes; ++c) + { + float prob = dets[j].prob[c]; + if (prob > 0) + cl = c; + } + std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; + } + TIMER_STOP + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + for (int i = 0; i < 3; i++) + rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; + + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for (int i = 0; i < 3; i++) + { + printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); + dnnType *out, *out_h; + int odim = out_dim[i].tot(); + readBinaryFile(output_bins[i], odim, &out_h, &out); + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} diff --git a/tests/yolo4/yolo4_512.cpp b/tests/yolo4/yolo4_512.cpp new file mode 100644 index 0000000..963df75 --- /dev/null +++ b/tests/yolo4/yolo4_512.cpp @@ -0,0 +1,666 @@ +#include +#include +#include "tkdnn.h" + +int main() +{ + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); + tk::dnn::Network net(dim); + + // create yolo4_512 model + std::string bin_path = "yolo4_512"; + int classes = 80; + tk::dnn::Yolo *yolo[3]; + + std::string input_bin = bin_path + "/layers/input.bin"; + + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin"}; + std::string c0_bin = bin_path + "/layers/c0.bin"; + std::string c1_bin = bin_path + "/layers/c1.bin"; + std::string c2_bin = bin_path + "/layers/c2.bin"; + std::string c3_bin = bin_path + "/layers/c3.bin"; + std::string c4_bin = bin_path + "/layers/c4.bin"; + std::string c5_bin = bin_path + "/layers/c5.bin"; + std::string c6_bin = bin_path + "/layers/c6.bin"; + std::string c7_bin = bin_path + "/layers/c7.bin"; + std::string c8_bin = bin_path + "/layers/c8.bin"; + std::string c10_bin = bin_path + "/layers/c10.bin"; + std::string c11_bin = bin_path + "/layers/c11.bin"; + std::string c12_bin = bin_path + "/layers/c12.bin"; + std::string c13_bin = bin_path + "/layers/c13.bin"; + std::string c14_bin = bin_path + "/layers/c14.bin"; + std::string c15_bin = bin_path + "/layers/c15.bin"; + std::string c16_bin = bin_path + "/layers/c16.bin"; + std::string c17_bin = bin_path + "/layers/c17.bin"; + std::string c18_bin = bin_path + "/layers/c18.bin"; + std::string c19_bin = bin_path + "/layers/c19.bin"; + std::string c20_bin = bin_path + "/layers/c20.bin"; + std::string c21_bin = bin_path + "/layers/c21.bin"; + std::string c23_bin = bin_path + "/layers/c23.bin"; + std::string c24_bin = bin_path + "/layers/c24.bin"; + std::string c25_bin = bin_path + "/layers/c25.bin"; + std::string c26_bin = bin_path + "/layers/c26.bin"; + std::string c27_bin = bin_path + "/layers/c27.bin"; + std::string c28_bin = bin_path + "/layers/c28.bin"; + std::string c29_bin = bin_path + "/layers/c29.bin"; + std::string c30_bin = bin_path + "/layers/c30.bin"; + std::string c31_bin = bin_path + "/layers/c31.bin"; + std::string c32_bin = bin_path + "/layers/c32.bin"; + std::string c33_bin = bin_path + "/layers/c33.bin"; + std::string c34_bin = bin_path + "/layers/c34.bin"; + std::string c35_bin = bin_path + "/layers/c35.bin"; + std::string c36_bin = bin_path + "/layers/c36.bin"; + std::string c37_bin = bin_path + "/layers/c37.bin"; + std::string c38_bin = bin_path + "/layers/c38.bin"; + std::string c39_bin = bin_path + "/layers/c39.bin"; + std::string c40_bin = bin_path + "/layers/c40.bin"; + std::string c41_bin = bin_path + "/layers/c41.bin"; + std::string c42_bin = bin_path + "/layers/c42.bin"; + std::string c43_bin = bin_path + "/layers/c43.bin"; + std::string c44_bin = bin_path + "/layers/c44.bin"; + std::string c45_bin = bin_path + "/layers/c45.bin"; + std::string c46_bin = bin_path + "/layers/c46.bin"; + std::string c47_bin = bin_path + "/layers/c47.bin"; + std::string c48_bin = bin_path + "/layers/c48.bin"; + std::string c49_bin = bin_path + "/layers/c49.bin"; + std::string c50_bin = bin_path + "/layers/c50.bin"; + std::string c51_bin = bin_path + "/layers/c51.bin"; + std::string c52_bin = bin_path + "/layers/c52.bin"; + std::string c53_bin = bin_path + "/layers/c53.bin"; + std::string c54_bin = bin_path + "/layers/c54.bin"; + std::string c55_bin = bin_path + "/layers/c55.bin"; + std::string c56_bin = bin_path + "/layers/c56.bin"; + std::string c57_bin = bin_path + "/layers/c57.bin"; + std::string c58_bin = bin_path + "/layers/c58.bin"; + std::string c59_bin = bin_path + "/layers/c59.bin"; + std::string c60_bin = bin_path + "/layers/c60.bin"; + std::string c61_bin = bin_path + "/layers/c61.bin"; + std::string c62_bin = bin_path + "/layers/c62.bin"; + std::string c63_bin = bin_path + "/layers/c63.bin"; + std::string c65_bin = bin_path + "/layers/c65.bin"; + std::string c66_bin = bin_path + "/layers/c66.bin"; + std::string c67_bin = bin_path + "/layers/c67.bin"; + std::string c68_bin = bin_path + "/layers/c68.bin"; + std::string c69_bin = bin_path + "/layers/c69.bin"; + std::string c70_bin = bin_path + "/layers/c70.bin"; + std::string c71_bin = bin_path + "/layers/c71.bin"; + std::string c72_bin = bin_path + "/layers/c72.bin"; + std::string c74_bin = bin_path + "/layers/c74.bin"; + std::string c75_bin = bin_path + "/layers/c75.bin"; + std::string c76_bin = bin_path + "/layers/c76.bin"; + std::string c77_bin = bin_path + "/layers/c77.bin"; + std::string c78_bin = bin_path + "/layers/c78.bin"; + std::string c80_bin = bin_path + "/layers/c80.bin"; + std::string c81_bin = bin_path + "/layers/c81.bin"; + std::string c82_bin = bin_path + "/layers/c82.bin"; + std::string c83_bin = bin_path + "/layers/c83.bin"; + std::string c85_bin = bin_path + "/layers/c85.bin"; + std::string c86_bin = bin_path + "/layers/c86.bin"; + std::string c87_bin = bin_path + "/layers/c87.bin"; + std::string c89_bin = bin_path + "/layers/c89.bin"; + std::string c90_bin = bin_path + "/layers/c90.bin"; + std::string c91_bin = bin_path + "/layers/c91.bin"; + std::string c92_bin = bin_path + "/layers/c92.bin"; + std::string c93_bin = bin_path + "/layers/c93.bin"; + std::string c94_bin = bin_path + "/layers/c94.bin"; + std::string c96_bin = bin_path + "/layers/c96.bin"; + std::string c97_bin = bin_path + "/layers/c97.bin"; + std::string c98_bin = bin_path + "/layers/c98.bin"; + std::string c99_bin = bin_path + "/layers/c99.bin"; + std::string c100_bin = bin_path + "/layers/c100.bin"; + std::string c101_bin = bin_path + "/layers/c101.bin"; + std::string c102_bin = bin_path + "/layers/c102.bin"; + std::string c103_bin = bin_path + "/layers/c103.bin"; + std::string c104_bin = bin_path + "/layers/c104.bin"; + std::string c105_bin = bin_path + "/layers/c105.bin"; + std::string c106_bin = bin_path + "/layers/c106.bin"; + std::string c107_bin = bin_path + "/layers/c107.bin"; + std::string c108_bin = bin_path + "/layers/c108.bin"; + std::string c109_bin = bin_path + "/layers/c109.bin"; + std::string c110_bin = bin_path + "/layers/c110.bin"; + std::string c111_bin = bin_path + "/layers/c111.bin"; + std::string c112_bin = bin_path + "/layers/c112.bin"; + std::string c113_bin = bin_path + "/layers/c113.bin"; + std::string c114_bin = bin_path + "/layers/c114.bin"; + std::string c115_bin = bin_path + "/layers/c115.bin"; + std::string c116_bin = bin_path + "/layers/c116.bin"; + std::string c117_bin = bin_path + "/layers/c117.bin"; + std::string c119_bin = bin_path + "/layers/c119.bin"; + std::string c120_bin = bin_path + "/layers/c120.bin"; + std::string c121_bin = bin_path + "/layers/c121.bin"; + std::string c122_bin = bin_path + "/layers/c122.bin"; + std::string c123_bin = bin_path + "/layers/c123.bin"; + std::string c124_bin = bin_path + "/layers/c124.bin"; + std::string c125_bin = bin_path + "/layers/c125.bin"; + std::string c126_bin = bin_path + "/layers/c126.bin"; + std::string c127_bin = bin_path + "/layers/c127.bin"; + std::string c128_bin = bin_path + "/layers/c128.bin"; + std::string c130_bin = bin_path + "/layers/c130.bin"; + std::string c131_bin = bin_path + "/layers/c131.bin"; + std::string c132_bin = bin_path + "/layers/c132.bin"; + std::string c133_bin = bin_path + "/layers/c133.bin"; + std::string c134_bin = bin_path + "/layers/c134.bin"; + std::string c135_bin = bin_path + "/layers/c135.bin"; + std::string c136_bin = bin_path + "/layers/c136.bin"; + std::string c137_bin = bin_path + "/layers/c137.bin"; + std::string c138_bin = bin_path + "/layers/c138.bin"; + std::string c141_bin = bin_path + "/layers/c141.bin"; + std::string c142_bin = bin_path + "/layers/c142.bin"; + std::string c143_bin = bin_path + "/layers/c143.bin"; + std::string c144_bin = bin_path + "/layers/c144.bin"; + std::string c145_bin = bin_path + "/layers/c145.bin"; + std::string c146_bin = bin_path + "/layers/c146.bin"; + std::string c147_bin = bin_path + "/layers/c147.bin"; + std::string c148_bin = bin_path + "/layers/c148.bin"; + std::string c149_bin = bin_path + "/layers/c149.bin"; + std::string c150_bin = bin_path + "/layers/c150.bin"; + std::string c151_bin = bin_path + "/layers/c151.bin"; + std::string c152_bin = bin_path + "/layers/c152.bin"; + std::string c153_bin = bin_path + "/layers/c153.bin"; + std::string c154_bin = bin_path + "/layers/c154.bin"; + std::string c155_bin = bin_path + "/layers/c155.bin"; + std::string c156_bin = bin_path + "/layers/c156.bin"; + std::string c157_bin = bin_path + "/layers/c157.bin"; + std::string c158_bin = bin_path + "/layers/c158.bin"; + std::string c159_bin = bin_path + "/layers/c159.bin"; + std::string c160_bin = bin_path + "/layers/c160.bin"; + std::string g139_bin = bin_path + "/layers/g139.bin"; + std::string g150_bin = bin_path + "/layers/g150.bin"; + std::string g161_bin = bin_path + "/layers/g161.bin"; + + + downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); + + tk::dnn::Conv2d c0(&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); + tk::dnn::Activation a0(&net, tk::dnn::ACTIVATION_MISH); + + // downsample + tk::dnn::Conv2d c1(&net, 64, 3, 3, 2, 2, 1, 1, c1_bin, true); + tk::dnn::Activation a1(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c2(&net, 64, 1, 1, 1, 1, 0, 0, c2_bin, true); + tk::dnn::Activation a2(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r3_layers[1] = {&a1}; + tk::dnn::Route r3(&net, r3_layers, 1); + + tk::dnn::Conv2d c4(&net, 64, 1, 1, 1, 1, 0, 0, c4_bin, true); + tk::dnn::Activation a4(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c5(&net, 32, 1, 1, 1, 1, 0, 0, c5_bin, true); + tk::dnn::Activation a5(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c6(&net, 64, 3, 3, 1, 1, 1, 1, c6_bin, true); + tk::dnn::Activation a6(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s7(&net, &a4); + + tk::dnn::Conv2d c8(&net, 64, 1, 1, 1, 1, 0, 0, c8_bin, true); + tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r9_layers[2] = {&a8, &a2}; + tk::dnn::Route r9(&net, r9_layers, 2); + + tk::dnn::Conv2d c10(&net, 64, 1, 1, 1, 1, 0, 0, c10_bin, true); + tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_MISH); + + // downsample + tk::dnn::Conv2d c11(&net, 128, 3, 3, 2, 2, 1, 1, c11_bin, true); + tk::dnn::Activation a11(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c12(&net, 64, 1, 1, 1, 1, 0, 0, c12_bin, true); + tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r13_layers[1] = {&a11}; + tk::dnn::Route r13(&net, r13_layers, 1); + + tk::dnn::Conv2d c14(&net, 64, 1, 1, 1, 1, 0, 0, c14_bin, true); + tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c15(&net, 64, 1, 1, 1, 1, 0, 0, c15_bin, true); + tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c16(&net, 64, 3, 3, 1, 1, 1, 1, c16_bin, true); + tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s17(&net, &a14); + + tk::dnn::Conv2d c18(&net, 64, 1, 1, 1, 1, 0, 0, c18_bin, true); + tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c19(&net, 64, 3, 3, 1, 1, 1, 1, c19_bin, true); + tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s20(&net, &s17); + + tk::dnn::Conv2d c21(&net, 64, 1, 1, 1, 1, 0, 0, c21_bin, true); + tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r22_layers[2] = {&a21, &a12}; + tk::dnn::Route r22(&net, r22_layers, 2); + + tk::dnn::Conv2d c23(&net, 128, 1, 1, 1, 1, 0, 0, c23_bin, true); + tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_MISH); + + //downsample + tk::dnn::Conv2d c24(&net, 256, 3, 3, 2, 2, 1, 1, c24_bin, true); + tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c25(&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true); + tk::dnn::Activation a25(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r26_layers[1] = {&a24}; + tk::dnn::Route r26(&net, r26_layers, 1); + + tk::dnn::Conv2d c27(&net, 128, 1, 1, 1, 1, 0, 0, c27_bin, true); + tk::dnn::Activation a27(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c28(&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true); + tk::dnn::Activation a28(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c29(&net, 128, 3, 3, 1, 1, 1, 1, c29_bin, true); + tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s30(&net, &a27); + + tk::dnn::Conv2d c31(&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true); + tk::dnn::Activation a31(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c32(&net, 128, 3, 3, 1, 1, 1, 1, c32_bin, true); + tk::dnn::Activation a32(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s33(&net, &s30); + + tk::dnn::Conv2d c34(&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true); + tk::dnn::Activation a34(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c35(&net, 128, 3, 3, 1, 1, 1, 1, c35_bin, true); + tk::dnn::Activation a35(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s36(&net, &s33); + + tk::dnn::Conv2d c37(&net, 128, 1, 1, 1, 1, 0, 0, c37_bin, true); + tk::dnn::Activation a37(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c38(&net, 128, 3, 3, 1, 1, 1, 1, c38_bin, true); + tk::dnn::Activation a38(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s39(&net, &s36); + + tk::dnn::Conv2d c40(&net, 128, 1, 1, 1, 1, 0, 0, c40_bin, true); + tk::dnn::Activation a40(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c41(&net, 128, 3, 3, 1, 1, 1, 1, c41_bin, true); + tk::dnn::Activation a41(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s42(&net, &s39); + + tk::dnn::Conv2d c43(&net, 128, 1, 1, 1, 1, 0, 0, c43_bin, true); + tk::dnn::Activation a43(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c44(&net, 128, 3, 3, 1, 1, 1, 1, c44_bin, true); + tk::dnn::Activation a44(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s45(&net, &s42); + + tk::dnn::Conv2d c46(&net, 128, 1, 1, 1, 1, 0, 0, c46_bin, true); + tk::dnn::Activation a46(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c47(&net, 128, 3, 3, 1, 1, 1, 1, c47_bin, true); + tk::dnn::Activation a47(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s48(&net, &s45); + + tk::dnn::Conv2d c49(&net, 128, 1, 1, 1, 1, 0, 0, c49_bin, true); + tk::dnn::Activation a49(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c50(&net, 128, 3, 3, 1, 1, 1, 1, c50_bin, true); + tk::dnn::Activation a50(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s51(&net, &s48); + + tk::dnn::Conv2d c52(&net, 128, 1, 1, 1, 1, 0, 0, c52_bin, true); + tk::dnn::Activation a52(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r53_layers[2] = {&a52, &a25}; + tk::dnn::Route r53(&net, r53_layers, 2); + + tk::dnn::Conv2d c54(&net, 256, 1, 1, 1, 1, 0, 0, c54_bin, true); + tk::dnn::Activation a54(&net, tk::dnn::ACTIVATION_MISH); + + //downsample + tk::dnn::Conv2d c55(&net, 512, 3, 3, 2, 2, 1, 1, c55_bin, true); + tk::dnn::Activation a55(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c56(&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true); + tk::dnn::Activation a56(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r57_layers[1] = {&a55}; + tk::dnn::Route r57(&net, r57_layers, 1); + + tk::dnn::Conv2d c58(&net, 256, 1, 1, 1, 1, 0, 0, c58_bin, true); + tk::dnn::Activation a58(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c59(&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true); + tk::dnn::Activation a59(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c60(&net, 256, 3, 3, 1, 1, 1, 1, c60_bin, true); + tk::dnn::Activation a60(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s61(&net, &a58); + + tk::dnn::Conv2d c62(&net, 256, 1, 1, 1, 1, 0, 0, c62_bin, true); + tk::dnn::Activation a62(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c63(&net, 256, 3, 3, 1, 1, 1, 1, c63_bin, true); + tk::dnn::Activation a63(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s64(&net, &s61); + + tk::dnn::Conv2d c65(&net, 256, 1, 1, 1, 1, 0, 0, c65_bin, true); + tk::dnn::Activation a65(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c66(&net, 256, 3, 3, 1, 1, 1, 1, c66_bin, true); + tk::dnn::Activation a66(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s67(&net, &s64); + + tk::dnn::Conv2d c68(&net, 256, 1, 1, 1, 1, 0, 0, c68_bin, true); + tk::dnn::Activation a68(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c69(&net, 256, 3, 3, 1, 1, 1, 1, c69_bin, true); + tk::dnn::Activation a69(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s70(&net, &s67); + + tk::dnn::Conv2d c71(&net, 256, 1, 1, 1, 1, 0, 0, c71_bin, true); + tk::dnn::Activation a71(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c72(&net, 256, 3, 3, 1, 1, 1, 1, c72_bin, true); + tk::dnn::Activation a72(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s73(&net, &s70); + + tk::dnn::Conv2d c74(&net, 256, 1, 1, 1, 1, 0, 0, c74_bin, true); + tk::dnn::Activation a74(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c75(&net, 256, 3, 3, 1, 1, 1, 1, c75_bin, true); + tk::dnn::Activation a75(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s76(&net, &s73); + + tk::dnn::Conv2d c77(&net, 256, 1, 1, 1, 1, 0, 0, c77_bin, true); + tk::dnn::Activation a77(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c78(&net, 256, 3, 3, 1, 1, 1, 1, c78_bin, true); + tk::dnn::Activation a78(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s79(&net, &s76); + + tk::dnn::Conv2d c80(&net, 256, 1, 1, 1, 1, 0, 0, c80_bin, true); + tk::dnn::Activation a80(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c81(&net, 256, 3, 3, 1, 1, 1, 1, c81_bin, true); + tk::dnn::Activation a81(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s82(&net, &s79); + + tk::dnn::Conv2d c83(&net, 256, 1, 1, 1, 1, 0, 0, c83_bin, true); + tk::dnn::Activation a83(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r84_layers[2] = {&a83, &a56}; + tk::dnn::Route r84(&net, r84_layers, 2); + + tk::dnn::Conv2d c85(&net, 512, 1, 1, 1, 1, 0, 0, c85_bin, true); + tk::dnn::Activation a85(&net, tk::dnn::ACTIVATION_MISH); + + //downsample + tk::dnn::Conv2d c86(&net, 1024, 3, 3, 2, 2, 1, 1, c86_bin, true); + tk::dnn::Activation a86(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c87(&net, 512, 1, 1, 1, 1, 0, 0, c87_bin, true); + tk::dnn::Activation a87(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r88_layers[1] = {&a86}; + tk::dnn::Route r88(&net, r88_layers, 1); + + tk::dnn::Conv2d c89(&net, 512, 1, 1, 1, 1, 0, 0, c89_bin, true); + tk::dnn::Activation a89(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c90(&net, 512, 1, 1, 1, 1, 0, 0, c90_bin, true); + tk::dnn::Activation a90(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c91(&net, 512, 3, 3, 1, 1, 1, 1, c91_bin, true); + tk::dnn::Activation a91(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s92(&net, &a89); + + tk::dnn::Conv2d c93(&net, 512, 1, 1, 1, 1, 0, 0, c93_bin, true); + tk::dnn::Activation a93(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c94(&net, 512, 3, 3, 1, 1, 1, 1, c94_bin, true); + tk::dnn::Activation a94(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s95(&net, &s92); + + tk::dnn::Conv2d c96(&net, 512, 1, 1, 1, 1, 0, 0, c96_bin, true); + tk::dnn::Activation a96(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c97(&net, 512, 3, 3, 1, 1, 1, 1, c97_bin, true); + tk::dnn::Activation a97(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s98(&net, &s95); + + tk::dnn::Conv2d c99(&net, 512, 1, 1, 1, 1, 0, 0, c99_bin, true); + tk::dnn::Activation a99(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c100(&net, 512, 3, 3, 1, 1, 1, 1, c100_bin, true); + tk::dnn::Activation a100(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s101(&net, &s98); + + tk::dnn::Conv2d c102(&net, 512, 1, 1, 1, 1, 0, 0, c102_bin, true); + tk::dnn::Activation a102(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r103_layers[2] = {&a102, &a87}; + tk::dnn::Route r103(&net, r103_layers, 2); + + tk::dnn::Conv2d c104(&net, 1024, 1, 1, 1, 1, 0, 0, c104_bin, true); + tk::dnn::Activation a104(&net, tk::dnn::ACTIVATION_MISH); + + + //################ + tk::dnn::Conv2d c105(&net, 512, 1, 1, 1, 1, 0, 0, c105_bin, true); + tk::dnn::Activation a105(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c106(&net, 1024, 3, 3, 1, 1, 1, 1, c106_bin, true); + tk::dnn::Activation a106(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c107(&net, 512, 1, 1, 1, 1, 0, 0, c107_bin, true); + tk::dnn::Activation a107(&net, tk::dnn::ACTIVATION_LEAKY); + + //SPP + tk::dnn::Pooling p108(&net, 5, 5, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); + tk::dnn::Layer *r109_layers[1] = {&a107}; + tk::dnn::Route r109(&net, r109_layers, 1); + + tk::dnn::Pooling p110(&net, 9, 9, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); + tk::dnn::Layer *r111_layers[1] = {&a107}; + tk::dnn::Route r111(&net, r111_layers, 1); + + tk::dnn::Pooling p112(&net, 13, 13, 1, 1, 12, 12, tk::dnn::POOLING_MAX_FIXEDSIZE); + tk::dnn::Layer *r113_layers[4] = {&p112, &p110, &p108, &a107}; + tk::dnn::Route r113(&net, r113_layers, 4); + //END SPP + + tk::dnn::Conv2d c114(&net, 512, 1, 1, 1, 1, 0, 0, c114_bin, true); + tk::dnn::Activation a114(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c115(&net, 1024, 3, 3, 1, 1, 1, 1, c115_bin, true); + tk::dnn::Activation a115(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c116(&net, 512, 1, 1, 1, 1, 0, 0, c116_bin, true); + tk::dnn::Activation a116(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c117(&net, 256, 1, 1, 1, 1, 0, 0, c117_bin, true); + tk::dnn::Activation a117(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Upsample u118(&net, 2); + tk::dnn::Layer *r119_layers[1] = {&a85}; + tk::dnn::Route r119(&net, r119_layers, 1); + tk::dnn::Conv2d c120(&net, 256, 1, 1, 1, 1, 0, 0, c120_bin, true); + tk::dnn::Activation a120(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r121_layers[2] = {&a120,&u118}; + tk::dnn::Route r121(&net, r121_layers, 2); + + tk::dnn::Conv2d c122(&net, 256, 1, 1, 1, 1, 0, 0, c122_bin, true); + tk::dnn::Activation a122(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c123(&net, 512, 3, 3, 1, 1, 1, 1, c123_bin, true); + tk::dnn::Activation a123(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c124(&net, 256, 1, 1, 1, 1, 0, 0, c124_bin, true); + tk::dnn::Activation a124(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c125(&net, 512, 3, 3, 1, 1, 1, 1, c125_bin, true); + tk::dnn::Activation a125(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c126(&net, 256, 1, 1, 1, 1, 0, 0, c126_bin, true); + tk::dnn::Activation a126(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c127(&net, 128, 1, 1, 1, 1, 0, 0, c127_bin, true); + tk::dnn::Activation a127(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Upsample u128(&net, 2); + tk::dnn::Layer *r129_layers[1] = {&a54}; + tk::dnn::Route r129(&net, r129_layers, 1); + tk::dnn::Conv2d c130(&net, 128, 1, 1, 1, 1, 0, 0, c130_bin, true); + tk::dnn::Activation a130(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r131_layers[2] = {&a130,&u128}; + tk::dnn::Route r131(&net, r131_layers, 2); + + + tk::dnn::Conv2d c132(&net, 128, 1, 1, 1, 1, 0, 0, c132_bin, true); + tk::dnn::Activation a132(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c133(&net, 256, 3, 3, 1, 1, 1, 1, c133_bin, true); + tk::dnn::Activation a133(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c134(&net, 128, 1, 1, 1, 1, 0, 0, c134_bin, true); + tk::dnn::Activation a134(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c135(&net, 256, 3, 3, 1, 1, 1, 1, c135_bin, true); + tk::dnn::Activation a135(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c136(&net, 128, 1, 1, 1, 1, 0, 0, c136_bin, true); + tk::dnn::Activation a136(&net, tk::dnn::ACTIVATION_LEAKY); + + + tk::dnn::Conv2d c137(&net, 256, 3, 3, 1, 1, 1, 1, c137_bin, true); + tk::dnn::Activation a137(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c138(&net, 255, 1, 1, 1, 1, 0, 0, c138_bin, false); + tk::dnn::Yolo yolo139(&net, classes, 3, g139_bin, 3, 1.2); + + tk::dnn::Layer *r140_layers[1] = {&a136}; + tk::dnn::Route r140(&net, r140_layers, 1); + tk::dnn::Conv2d c141(&net, 256, 3, 3, 2, 2, 1, 1, c141_bin, true); + tk::dnn::Activation a141(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r142_layers[2] = {&a141,&a126}; + tk::dnn::Route r142(&net, r142_layers, 2); + + tk::dnn::Conv2d c143(&net, 256, 1, 1, 1, 1, 0, 0, c143_bin, true); + tk::dnn::Activation a143(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c144(&net, 512, 3, 3, 1, 1, 1, 1, c144_bin, true); + tk::dnn::Activation a144(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c145(&net, 256, 1, 1, 1, 1, 0, 0, c145_bin, true); + tk::dnn::Activation a145(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c146(&net, 512, 3, 3, 1, 1, 1, 1, c146_bin, true); + tk::dnn::Activation a146(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c147(&net, 256, 1, 1, 1, 1, 0, 0, c147_bin, true); + tk::dnn::Activation a147(&net, tk::dnn::ACTIVATION_LEAKY); + + tk::dnn::Conv2d c148(&net, 512, 3, 3, 1, 1, 1, 1, c148_bin, true); + tk::dnn::Activation a148(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c149(&net, 255, 1, 1, 1, 1, 0, 0, c149_bin, false); + tk::dnn::Yolo yolo150(&net, classes, 3, g150_bin, 3, 1.1); + + tk::dnn::Layer *r151_layers[1] = {&a147}; + tk::dnn::Route r151(&net, r151_layers, 1); + tk::dnn::Conv2d c152(&net, 512, 3, 3, 2, 2, 1, 1, c152_bin, true); + tk::dnn::Activation a152(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r153_layers[2] = {&a152,&a116}; + tk::dnn::Route r153(&net, r153_layers, 2); + + tk::dnn::Conv2d c154(&net, 512, 1, 1, 1, 1, 0, 0, c154_bin, true); + tk::dnn::Activation a154(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c155(&net, 1024, 3, 3, 1, 1, 1, 1, c155_bin, true); + tk::dnn::Activation a155(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c156(&net, 512, 1, 1, 1, 1, 0, 0, c156_bin, true); + tk::dnn::Activation a156(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c157(&net, 1024, 3, 3, 1, 1, 1, 1, c157_bin, true); + tk::dnn::Activation a157(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c158(&net, 512, 1, 1, 1, 1, 0, 0, c158_bin, true); + tk::dnn::Activation a158(&net, tk::dnn::ACTIVATION_LEAKY); + + tk::dnn::Conv2d c159(&net, 1024, 3, 3, 1, 1, 1, 1, c159_bin, true); + tk::dnn::Activation a159(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c160(&net, 255, 1, 1, 1, 1, 0, 0, c160_bin, false); + tk::dnn::Yolo yolo161(&net, classes, 3, g161_bin, 3, 1.05); + + + + + + + yolo[0] = &yolo139; + yolo[1] = &yolo150; + yolo[2] = &yolo161; + + // fill classes names + for (int i = 0; i < 3; i++) + { + yolo[i]->classesNames = {"person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"}; + } + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + + //print network model + net.print(); + + // //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo4_512")); + + // the network have 3 outputs + tk::dnn::dataDim_t out_dim[3]; + for (int i = 0; i < 3; i++) + out_dim[i] = yolo[i]->output_dim; + dnnType *cudnn_out[3], *rt_out[3]; + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + + for (int i = 0; i < 3; i++) + cudnn_out[i] = yolo[i]->dstData; + + printCenteredTitle(" compute detections ", '=', 30); + TIMER_START + int ndets = 0; + tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); + for (int i = 0; i < 3; i++) + yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); + tk::dnn::Yolo::mergeDetections(dets, ndets, classes); + + for (int j = 0; j < ndets; j++) + { + tk::dnn::Yolo::box b = dets[j].bbox; + int x0 = (b.x - b.w / 2.); + int x1 = (b.x + b.w / 2.); + int y0 = (b.y - b.h / 2.); + int y1 = (b.y + b.h / 2.); + + int cl = 0; + for (int c = 0; c < classes; ++c) + { + float prob = dets[j].prob[c]; + if (prob > 0) + cl = c; + } + std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; + } + TIMER_STOP + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + for (int i = 0; i < 3; i++) + rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; + + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for (int i = 0; i < 3; i++) + { + printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); + dnnType *out, *out_h; + int odim = out_dim[i].tot(); + readBinaryFile(output_bins[i], odim, &out_h, &out); + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} diff --git a/tests/yolo4/yolo4_608.cpp b/tests/yolo4/yolo4_608.cpp new file mode 100644 index 0000000..64d8f34 --- /dev/null +++ b/tests/yolo4/yolo4_608.cpp @@ -0,0 +1,666 @@ +#include +#include +#include "tkdnn.h" + +int main() +{ + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 608, 608, 1); + tk::dnn::Network net(dim); + + // create yolo4_608 model + std::string bin_path = "yolo4_608"; + int classes = 80; + tk::dnn::Yolo *yolo[3]; + + std::string input_bin = bin_path + "/layers/input.bin"; + + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin"}; + std::string c0_bin = bin_path + "/layers/c0.bin"; + std::string c1_bin = bin_path + "/layers/c1.bin"; + std::string c2_bin = bin_path + "/layers/c2.bin"; + std::string c3_bin = bin_path + "/layers/c3.bin"; + std::string c4_bin = bin_path + "/layers/c4.bin"; + std::string c5_bin = bin_path + "/layers/c5.bin"; + std::string c6_bin = bin_path + "/layers/c6.bin"; + std::string c7_bin = bin_path + "/layers/c7.bin"; + std::string c8_bin = bin_path + "/layers/c8.bin"; + std::string c10_bin = bin_path + "/layers/c10.bin"; + std::string c11_bin = bin_path + "/layers/c11.bin"; + std::string c12_bin = bin_path + "/layers/c12.bin"; + std::string c13_bin = bin_path + "/layers/c13.bin"; + std::string c14_bin = bin_path + "/layers/c14.bin"; + std::string c15_bin = bin_path + "/layers/c15.bin"; + std::string c16_bin = bin_path + "/layers/c16.bin"; + std::string c17_bin = bin_path + "/layers/c17.bin"; + std::string c18_bin = bin_path + "/layers/c18.bin"; + std::string c19_bin = bin_path + "/layers/c19.bin"; + std::string c20_bin = bin_path + "/layers/c20.bin"; + std::string c21_bin = bin_path + "/layers/c21.bin"; + std::string c23_bin = bin_path + "/layers/c23.bin"; + std::string c24_bin = bin_path + "/layers/c24.bin"; + std::string c25_bin = bin_path + "/layers/c25.bin"; + std::string c26_bin = bin_path + "/layers/c26.bin"; + std::string c27_bin = bin_path + "/layers/c27.bin"; + std::string c28_bin = bin_path + "/layers/c28.bin"; + std::string c29_bin = bin_path + "/layers/c29.bin"; + std::string c30_bin = bin_path + "/layers/c30.bin"; + std::string c31_bin = bin_path + "/layers/c31.bin"; + std::string c32_bin = bin_path + "/layers/c32.bin"; + std::string c33_bin = bin_path + "/layers/c33.bin"; + std::string c34_bin = bin_path + "/layers/c34.bin"; + std::string c35_bin = bin_path + "/layers/c35.bin"; + std::string c36_bin = bin_path + "/layers/c36.bin"; + std::string c37_bin = bin_path + "/layers/c37.bin"; + std::string c38_bin = bin_path + "/layers/c38.bin"; + std::string c39_bin = bin_path + "/layers/c39.bin"; + std::string c40_bin = bin_path + "/layers/c40.bin"; + std::string c41_bin = bin_path + "/layers/c41.bin"; + std::string c42_bin = bin_path + "/layers/c42.bin"; + std::string c43_bin = bin_path + "/layers/c43.bin"; + std::string c44_bin = bin_path + "/layers/c44.bin"; + std::string c45_bin = bin_path + "/layers/c45.bin"; + std::string c46_bin = bin_path + "/layers/c46.bin"; + std::string c47_bin = bin_path + "/layers/c47.bin"; + std::string c48_bin = bin_path + "/layers/c48.bin"; + std::string c49_bin = bin_path + "/layers/c49.bin"; + std::string c50_bin = bin_path + "/layers/c50.bin"; + std::string c51_bin = bin_path + "/layers/c51.bin"; + std::string c52_bin = bin_path + "/layers/c52.bin"; + std::string c53_bin = bin_path + "/layers/c53.bin"; + std::string c54_bin = bin_path + "/layers/c54.bin"; + std::string c55_bin = bin_path + "/layers/c55.bin"; + std::string c56_bin = bin_path + "/layers/c56.bin"; + std::string c57_bin = bin_path + "/layers/c57.bin"; + std::string c58_bin = bin_path + "/layers/c58.bin"; + std::string c59_bin = bin_path + "/layers/c59.bin"; + std::string c60_bin = bin_path + "/layers/c60.bin"; + std::string c61_bin = bin_path + "/layers/c61.bin"; + std::string c62_bin = bin_path + "/layers/c62.bin"; + std::string c63_bin = bin_path + "/layers/c63.bin"; + std::string c65_bin = bin_path + "/layers/c65.bin"; + std::string c66_bin = bin_path + "/layers/c66.bin"; + std::string c67_bin = bin_path + "/layers/c67.bin"; + std::string c68_bin = bin_path + "/layers/c68.bin"; + std::string c69_bin = bin_path + "/layers/c69.bin"; + std::string c70_bin = bin_path + "/layers/c70.bin"; + std::string c71_bin = bin_path + "/layers/c71.bin"; + std::string c72_bin = bin_path + "/layers/c72.bin"; + std::string c74_bin = bin_path + "/layers/c74.bin"; + std::string c75_bin = bin_path + "/layers/c75.bin"; + std::string c76_bin = bin_path + "/layers/c76.bin"; + std::string c77_bin = bin_path + "/layers/c77.bin"; + std::string c78_bin = bin_path + "/layers/c78.bin"; + std::string c80_bin = bin_path + "/layers/c80.bin"; + std::string c81_bin = bin_path + "/layers/c81.bin"; + std::string c82_bin = bin_path + "/layers/c82.bin"; + std::string c83_bin = bin_path + "/layers/c83.bin"; + std::string c85_bin = bin_path + "/layers/c85.bin"; + std::string c86_bin = bin_path + "/layers/c86.bin"; + std::string c87_bin = bin_path + "/layers/c87.bin"; + std::string c89_bin = bin_path + "/layers/c89.bin"; + std::string c90_bin = bin_path + "/layers/c90.bin"; + std::string c91_bin = bin_path + "/layers/c91.bin"; + std::string c92_bin = bin_path + "/layers/c92.bin"; + std::string c93_bin = bin_path + "/layers/c93.bin"; + std::string c94_bin = bin_path + "/layers/c94.bin"; + std::string c96_bin = bin_path + "/layers/c96.bin"; + std::string c97_bin = bin_path + "/layers/c97.bin"; + std::string c98_bin = bin_path + "/layers/c98.bin"; + std::string c99_bin = bin_path + "/layers/c99.bin"; + std::string c100_bin = bin_path + "/layers/c100.bin"; + std::string c101_bin = bin_path + "/layers/c101.bin"; + std::string c102_bin = bin_path + "/layers/c102.bin"; + std::string c103_bin = bin_path + "/layers/c103.bin"; + std::string c104_bin = bin_path + "/layers/c104.bin"; + std::string c105_bin = bin_path + "/layers/c105.bin"; + std::string c106_bin = bin_path + "/layers/c106.bin"; + std::string c107_bin = bin_path + "/layers/c107.bin"; + std::string c108_bin = bin_path + "/layers/c108.bin"; + std::string c109_bin = bin_path + "/layers/c109.bin"; + std::string c110_bin = bin_path + "/layers/c110.bin"; + std::string c111_bin = bin_path + "/layers/c111.bin"; + std::string c112_bin = bin_path + "/layers/c112.bin"; + std::string c113_bin = bin_path + "/layers/c113.bin"; + std::string c114_bin = bin_path + "/layers/c114.bin"; + std::string c115_bin = bin_path + "/layers/c115.bin"; + std::string c116_bin = bin_path + "/layers/c116.bin"; + std::string c117_bin = bin_path + "/layers/c117.bin"; + std::string c119_bin = bin_path + "/layers/c119.bin"; + std::string c120_bin = bin_path + "/layers/c120.bin"; + std::string c121_bin = bin_path + "/layers/c121.bin"; + std::string c122_bin = bin_path + "/layers/c122.bin"; + std::string c123_bin = bin_path + "/layers/c123.bin"; + std::string c124_bin = bin_path + "/layers/c124.bin"; + std::string c125_bin = bin_path + "/layers/c125.bin"; + std::string c126_bin = bin_path + "/layers/c126.bin"; + std::string c127_bin = bin_path + "/layers/c127.bin"; + std::string c128_bin = bin_path + "/layers/c128.bin"; + std::string c130_bin = bin_path + "/layers/c130.bin"; + std::string c131_bin = bin_path + "/layers/c131.bin"; + std::string c132_bin = bin_path + "/layers/c132.bin"; + std::string c133_bin = bin_path + "/layers/c133.bin"; + std::string c134_bin = bin_path + "/layers/c134.bin"; + std::string c135_bin = bin_path + "/layers/c135.bin"; + std::string c136_bin = bin_path + "/layers/c136.bin"; + std::string c137_bin = bin_path + "/layers/c137.bin"; + std::string c138_bin = bin_path + "/layers/c138.bin"; + std::string c141_bin = bin_path + "/layers/c141.bin"; + std::string c142_bin = bin_path + "/layers/c142.bin"; + std::string c143_bin = bin_path + "/layers/c143.bin"; + std::string c144_bin = bin_path + "/layers/c144.bin"; + std::string c145_bin = bin_path + "/layers/c145.bin"; + std::string c146_bin = bin_path + "/layers/c146.bin"; + std::string c147_bin = bin_path + "/layers/c147.bin"; + std::string c148_bin = bin_path + "/layers/c148.bin"; + std::string c149_bin = bin_path + "/layers/c149.bin"; + std::string c150_bin = bin_path + "/layers/c150.bin"; + std::string c151_bin = bin_path + "/layers/c151.bin"; + std::string c152_bin = bin_path + "/layers/c152.bin"; + std::string c153_bin = bin_path + "/layers/c153.bin"; + std::string c154_bin = bin_path + "/layers/c154.bin"; + std::string c155_bin = bin_path + "/layers/c155.bin"; + std::string c156_bin = bin_path + "/layers/c156.bin"; + std::string c157_bin = bin_path + "/layers/c157.bin"; + std::string c158_bin = bin_path + "/layers/c158.bin"; + std::string c159_bin = bin_path + "/layers/c159.bin"; + std::string c160_bin = bin_path + "/layers/c160.bin"; + std::string g139_bin = bin_path + "/layers/g139.bin"; + std::string g150_bin = bin_path + "/layers/g150.bin"; + std::string g161_bin = bin_path + "/layers/g161.bin"; + + + downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); + + tk::dnn::Conv2d c0(&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); + tk::dnn::Activation a0(&net, tk::dnn::ACTIVATION_MISH); + + // downsample + tk::dnn::Conv2d c1(&net, 64, 3, 3, 2, 2, 1, 1, c1_bin, true); + tk::dnn::Activation a1(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c2(&net, 64, 1, 1, 1, 1, 0, 0, c2_bin, true); + tk::dnn::Activation a2(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r3_layers[1] = {&a1}; + tk::dnn::Route r3(&net, r3_layers, 1); + + tk::dnn::Conv2d c4(&net, 64, 1, 1, 1, 1, 0, 0, c4_bin, true); + tk::dnn::Activation a4(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c5(&net, 32, 1, 1, 1, 1, 0, 0, c5_bin, true); + tk::dnn::Activation a5(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c6(&net, 64, 3, 3, 1, 1, 1, 1, c6_bin, true); + tk::dnn::Activation a6(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s7(&net, &a4); + + tk::dnn::Conv2d c8(&net, 64, 1, 1, 1, 1, 0, 0, c8_bin, true); + tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r9_layers[2] = {&a8, &a2}; + tk::dnn::Route r9(&net, r9_layers, 2); + + tk::dnn::Conv2d c10(&net, 64, 1, 1, 1, 1, 0, 0, c10_bin, true); + tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_MISH); + + // downsample + tk::dnn::Conv2d c11(&net, 128, 3, 3, 2, 2, 1, 1, c11_bin, true); + tk::dnn::Activation a11(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c12(&net, 64, 1, 1, 1, 1, 0, 0, c12_bin, true); + tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r13_layers[1] = {&a11}; + tk::dnn::Route r13(&net, r13_layers, 1); + + tk::dnn::Conv2d c14(&net, 64, 1, 1, 1, 1, 0, 0, c14_bin, true); + tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c15(&net, 64, 1, 1, 1, 1, 0, 0, c15_bin, true); + tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c16(&net, 64, 3, 3, 1, 1, 1, 1, c16_bin, true); + tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s17(&net, &a14); + + tk::dnn::Conv2d c18(&net, 64, 1, 1, 1, 1, 0, 0, c18_bin, true); + tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c19(&net, 64, 3, 3, 1, 1, 1, 1, c19_bin, true); + tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s20(&net, &s17); + + tk::dnn::Conv2d c21(&net, 64, 1, 1, 1, 1, 0, 0, c21_bin, true); + tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r22_layers[2] = {&a21, &a12}; + tk::dnn::Route r22(&net, r22_layers, 2); + + tk::dnn::Conv2d c23(&net, 128, 1, 1, 1, 1, 0, 0, c23_bin, true); + tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_MISH); + + //downsample + tk::dnn::Conv2d c24(&net, 256, 3, 3, 2, 2, 1, 1, c24_bin, true); + tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c25(&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true); + tk::dnn::Activation a25(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r26_layers[1] = {&a24}; + tk::dnn::Route r26(&net, r26_layers, 1); + + tk::dnn::Conv2d c27(&net, 128, 1, 1, 1, 1, 0, 0, c27_bin, true); + tk::dnn::Activation a27(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c28(&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true); + tk::dnn::Activation a28(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c29(&net, 128, 3, 3, 1, 1, 1, 1, c29_bin, true); + tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s30(&net, &a27); + + tk::dnn::Conv2d c31(&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true); + tk::dnn::Activation a31(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c32(&net, 128, 3, 3, 1, 1, 1, 1, c32_bin, true); + tk::dnn::Activation a32(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s33(&net, &s30); + + tk::dnn::Conv2d c34(&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true); + tk::dnn::Activation a34(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c35(&net, 128, 3, 3, 1, 1, 1, 1, c35_bin, true); + tk::dnn::Activation a35(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s36(&net, &s33); + + tk::dnn::Conv2d c37(&net, 128, 1, 1, 1, 1, 0, 0, c37_bin, true); + tk::dnn::Activation a37(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c38(&net, 128, 3, 3, 1, 1, 1, 1, c38_bin, true); + tk::dnn::Activation a38(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s39(&net, &s36); + + tk::dnn::Conv2d c40(&net, 128, 1, 1, 1, 1, 0, 0, c40_bin, true); + tk::dnn::Activation a40(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c41(&net, 128, 3, 3, 1, 1, 1, 1, c41_bin, true); + tk::dnn::Activation a41(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s42(&net, &s39); + + tk::dnn::Conv2d c43(&net, 128, 1, 1, 1, 1, 0, 0, c43_bin, true); + tk::dnn::Activation a43(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c44(&net, 128, 3, 3, 1, 1, 1, 1, c44_bin, true); + tk::dnn::Activation a44(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s45(&net, &s42); + + tk::dnn::Conv2d c46(&net, 128, 1, 1, 1, 1, 0, 0, c46_bin, true); + tk::dnn::Activation a46(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c47(&net, 128, 3, 3, 1, 1, 1, 1, c47_bin, true); + tk::dnn::Activation a47(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s48(&net, &s45); + + tk::dnn::Conv2d c49(&net, 128, 1, 1, 1, 1, 0, 0, c49_bin, true); + tk::dnn::Activation a49(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c50(&net, 128, 3, 3, 1, 1, 1, 1, c50_bin, true); + tk::dnn::Activation a50(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s51(&net, &s48); + + tk::dnn::Conv2d c52(&net, 128, 1, 1, 1, 1, 0, 0, c52_bin, true); + tk::dnn::Activation a52(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r53_layers[2] = {&a52, &a25}; + tk::dnn::Route r53(&net, r53_layers, 2); + + tk::dnn::Conv2d c54(&net, 256, 1, 1, 1, 1, 0, 0, c54_bin, true); + tk::dnn::Activation a54(&net, tk::dnn::ACTIVATION_MISH); + + //downsample + tk::dnn::Conv2d c55(&net, 512, 3, 3, 2, 2, 1, 1, c55_bin, true); + tk::dnn::Activation a55(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c56(&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true); + tk::dnn::Activation a56(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r57_layers[1] = {&a55}; + tk::dnn::Route r57(&net, r57_layers, 1); + + tk::dnn::Conv2d c58(&net, 256, 1, 1, 1, 1, 0, 0, c58_bin, true); + tk::dnn::Activation a58(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c59(&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true); + tk::dnn::Activation a59(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c60(&net, 256, 3, 3, 1, 1, 1, 1, c60_bin, true); + tk::dnn::Activation a60(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s61(&net, &a58); + + tk::dnn::Conv2d c62(&net, 256, 1, 1, 1, 1, 0, 0, c62_bin, true); + tk::dnn::Activation a62(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c63(&net, 256, 3, 3, 1, 1, 1, 1, c63_bin, true); + tk::dnn::Activation a63(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s64(&net, &s61); + + tk::dnn::Conv2d c65(&net, 256, 1, 1, 1, 1, 0, 0, c65_bin, true); + tk::dnn::Activation a65(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c66(&net, 256, 3, 3, 1, 1, 1, 1, c66_bin, true); + tk::dnn::Activation a66(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s67(&net, &s64); + + tk::dnn::Conv2d c68(&net, 256, 1, 1, 1, 1, 0, 0, c68_bin, true); + tk::dnn::Activation a68(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c69(&net, 256, 3, 3, 1, 1, 1, 1, c69_bin, true); + tk::dnn::Activation a69(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s70(&net, &s67); + + tk::dnn::Conv2d c71(&net, 256, 1, 1, 1, 1, 0, 0, c71_bin, true); + tk::dnn::Activation a71(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c72(&net, 256, 3, 3, 1, 1, 1, 1, c72_bin, true); + tk::dnn::Activation a72(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s73(&net, &s70); + + tk::dnn::Conv2d c74(&net, 256, 1, 1, 1, 1, 0, 0, c74_bin, true); + tk::dnn::Activation a74(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c75(&net, 256, 3, 3, 1, 1, 1, 1, c75_bin, true); + tk::dnn::Activation a75(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s76(&net, &s73); + + tk::dnn::Conv2d c77(&net, 256, 1, 1, 1, 1, 0, 0, c77_bin, true); + tk::dnn::Activation a77(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c78(&net, 256, 3, 3, 1, 1, 1, 1, c78_bin, true); + tk::dnn::Activation a78(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s79(&net, &s76); + + tk::dnn::Conv2d c80(&net, 256, 1, 1, 1, 1, 0, 0, c80_bin, true); + tk::dnn::Activation a80(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c81(&net, 256, 3, 3, 1, 1, 1, 1, c81_bin, true); + tk::dnn::Activation a81(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s82(&net, &s79); + + tk::dnn::Conv2d c83(&net, 256, 1, 1, 1, 1, 0, 0, c83_bin, true); + tk::dnn::Activation a83(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r84_layers[2] = {&a83, &a56}; + tk::dnn::Route r84(&net, r84_layers, 2); + + tk::dnn::Conv2d c85(&net, 512, 1, 1, 1, 1, 0, 0, c85_bin, true); + tk::dnn::Activation a85(&net, tk::dnn::ACTIVATION_MISH); + + //downsample + tk::dnn::Conv2d c86(&net, 1024, 3, 3, 2, 2, 1, 1, c86_bin, true); + tk::dnn::Activation a86(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c87(&net, 512, 1, 1, 1, 1, 0, 0, c87_bin, true); + tk::dnn::Activation a87(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r88_layers[1] = {&a86}; + tk::dnn::Route r88(&net, r88_layers, 1); + + tk::dnn::Conv2d c89(&net, 512, 1, 1, 1, 1, 0, 0, c89_bin, true); + tk::dnn::Activation a89(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c90(&net, 512, 1, 1, 1, 1, 0, 0, c90_bin, true); + tk::dnn::Activation a90(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c91(&net, 512, 3, 3, 1, 1, 1, 1, c91_bin, true); + tk::dnn::Activation a91(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s92(&net, &a89); + + tk::dnn::Conv2d c93(&net, 512, 1, 1, 1, 1, 0, 0, c93_bin, true); + tk::dnn::Activation a93(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c94(&net, 512, 3, 3, 1, 1, 1, 1, c94_bin, true); + tk::dnn::Activation a94(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s95(&net, &s92); + + tk::dnn::Conv2d c96(&net, 512, 1, 1, 1, 1, 0, 0, c96_bin, true); + tk::dnn::Activation a96(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c97(&net, 512, 3, 3, 1, 1, 1, 1, c97_bin, true); + tk::dnn::Activation a97(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s98(&net, &s95); + + tk::dnn::Conv2d c99(&net, 512, 1, 1, 1, 1, 0, 0, c99_bin, true); + tk::dnn::Activation a99(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c100(&net, 512, 3, 3, 1, 1, 1, 1, c100_bin, true); + tk::dnn::Activation a100(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s101(&net, &s98); + + tk::dnn::Conv2d c102(&net, 512, 1, 1, 1, 1, 0, 0, c102_bin, true); + tk::dnn::Activation a102(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r103_layers[2] = {&a102, &a87}; + tk::dnn::Route r103(&net, r103_layers, 2); + + tk::dnn::Conv2d c104(&net, 1024, 1, 1, 1, 1, 0, 0, c104_bin, true); + tk::dnn::Activation a104(&net, tk::dnn::ACTIVATION_MISH); + + + //################ + tk::dnn::Conv2d c105(&net, 512, 1, 1, 1, 1, 0, 0, c105_bin, true); + tk::dnn::Activation a105(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c106(&net, 1024, 3, 3, 1, 1, 1, 1, c106_bin, true); + tk::dnn::Activation a106(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c107(&net, 512, 1, 1, 1, 1, 0, 0, c107_bin, true); + tk::dnn::Activation a107(&net, tk::dnn::ACTIVATION_LEAKY); + + //SPP + tk::dnn::Pooling p108(&net, 5, 5, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); + tk::dnn::Layer *r109_layers[1] = {&a107}; + tk::dnn::Route r109(&net, r109_layers, 1); + + tk::dnn::Pooling p110(&net, 9, 9, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); + tk::dnn::Layer *r111_layers[1] = {&a107}; + tk::dnn::Route r111(&net, r111_layers, 1); + + tk::dnn::Pooling p112(&net, 13, 13, 1, 1, 12, 12, tk::dnn::POOLING_MAX_FIXEDSIZE); + tk::dnn::Layer *r113_layers[4] = {&p112, &p110, &p108, &a107}; + tk::dnn::Route r113(&net, r113_layers, 4); + //END SPP + + tk::dnn::Conv2d c114(&net, 512, 1, 1, 1, 1, 0, 0, c114_bin, true); + tk::dnn::Activation a114(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c115(&net, 1024, 3, 3, 1, 1, 1, 1, c115_bin, true); + tk::dnn::Activation a115(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c116(&net, 512, 1, 1, 1, 1, 0, 0, c116_bin, true); + tk::dnn::Activation a116(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c117(&net, 256, 1, 1, 1, 1, 0, 0, c117_bin, true); + tk::dnn::Activation a117(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Upsample u118(&net, 2); + tk::dnn::Layer *r119_layers[1] = {&a85}; + tk::dnn::Route r119(&net, r119_layers, 1); + tk::dnn::Conv2d c120(&net, 256, 1, 1, 1, 1, 0, 0, c120_bin, true); + tk::dnn::Activation a120(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r121_layers[2] = {&a120,&u118}; + tk::dnn::Route r121(&net, r121_layers, 2); + + tk::dnn::Conv2d c122(&net, 256, 1, 1, 1, 1, 0, 0, c122_bin, true); + tk::dnn::Activation a122(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c123(&net, 512, 3, 3, 1, 1, 1, 1, c123_bin, true); + tk::dnn::Activation a123(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c124(&net, 256, 1, 1, 1, 1, 0, 0, c124_bin, true); + tk::dnn::Activation a124(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c125(&net, 512, 3, 3, 1, 1, 1, 1, c125_bin, true); + tk::dnn::Activation a125(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c126(&net, 256, 1, 1, 1, 1, 0, 0, c126_bin, true); + tk::dnn::Activation a126(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c127(&net, 128, 1, 1, 1, 1, 0, 0, c127_bin, true); + tk::dnn::Activation a127(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Upsample u128(&net, 2); + tk::dnn::Layer *r129_layers[1] = {&a54}; + tk::dnn::Route r129(&net, r129_layers, 1); + tk::dnn::Conv2d c130(&net, 128, 1, 1, 1, 1, 0, 0, c130_bin, true); + tk::dnn::Activation a130(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r131_layers[2] = {&a130,&u128}; + tk::dnn::Route r131(&net, r131_layers, 2); + + + tk::dnn::Conv2d c132(&net, 128, 1, 1, 1, 1, 0, 0, c132_bin, true); + tk::dnn::Activation a132(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c133(&net, 256, 3, 3, 1, 1, 1, 1, c133_bin, true); + tk::dnn::Activation a133(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c134(&net, 128, 1, 1, 1, 1, 0, 0, c134_bin, true); + tk::dnn::Activation a134(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c135(&net, 256, 3, 3, 1, 1, 1, 1, c135_bin, true); + tk::dnn::Activation a135(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c136(&net, 128, 1, 1, 1, 1, 0, 0, c136_bin, true); + tk::dnn::Activation a136(&net, tk::dnn::ACTIVATION_LEAKY); + + + tk::dnn::Conv2d c137(&net, 256, 3, 3, 1, 1, 1, 1, c137_bin, true); + tk::dnn::Activation a137(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c138(&net, 255, 1, 1, 1, 1, 0, 0, c138_bin, false); + tk::dnn::Yolo yolo139(&net, classes, 3, g139_bin, 3, 1.2); + + tk::dnn::Layer *r140_layers[1] = {&a136}; + tk::dnn::Route r140(&net, r140_layers, 1); + tk::dnn::Conv2d c141(&net, 256, 3, 3, 2, 2, 1, 1, c141_bin, true); + tk::dnn::Activation a141(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r142_layers[2] = {&a141,&a126}; + tk::dnn::Route r142(&net, r142_layers, 2); + + tk::dnn::Conv2d c143(&net, 256, 1, 1, 1, 1, 0, 0, c143_bin, true); + tk::dnn::Activation a143(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c144(&net, 512, 3, 3, 1, 1, 1, 1, c144_bin, true); + tk::dnn::Activation a144(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c145(&net, 256, 1, 1, 1, 1, 0, 0, c145_bin, true); + tk::dnn::Activation a145(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c146(&net, 512, 3, 3, 1, 1, 1, 1, c146_bin, true); + tk::dnn::Activation a146(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c147(&net, 256, 1, 1, 1, 1, 0, 0, c147_bin, true); + tk::dnn::Activation a147(&net, tk::dnn::ACTIVATION_LEAKY); + + tk::dnn::Conv2d c148(&net, 512, 3, 3, 1, 1, 1, 1, c148_bin, true); + tk::dnn::Activation a148(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c149(&net, 255, 1, 1, 1, 1, 0, 0, c149_bin, false); + tk::dnn::Yolo yolo150(&net, classes, 3, g150_bin, 3, 1.1); + + tk::dnn::Layer *r151_layers[1] = {&a147}; + tk::dnn::Route r151(&net, r151_layers, 1); + tk::dnn::Conv2d c152(&net, 512, 3, 3, 2, 2, 1, 1, c152_bin, true); + tk::dnn::Activation a152(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r153_layers[2] = {&a152,&a116}; + tk::dnn::Route r153(&net, r153_layers, 2); + + tk::dnn::Conv2d c154(&net, 512, 1, 1, 1, 1, 0, 0, c154_bin, true); + tk::dnn::Activation a154(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c155(&net, 1024, 3, 3, 1, 1, 1, 1, c155_bin, true); + tk::dnn::Activation a155(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c156(&net, 512, 1, 1, 1, 1, 0, 0, c156_bin, true); + tk::dnn::Activation a156(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c157(&net, 1024, 3, 3, 1, 1, 1, 1, c157_bin, true); + tk::dnn::Activation a157(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c158(&net, 512, 1, 1, 1, 1, 0, 0, c158_bin, true); + tk::dnn::Activation a158(&net, tk::dnn::ACTIVATION_LEAKY); + + tk::dnn::Conv2d c159(&net, 1024, 3, 3, 1, 1, 1, 1, c159_bin, true); + tk::dnn::Activation a159(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c160(&net, 255, 1, 1, 1, 1, 0, 0, c160_bin, false); + tk::dnn::Yolo yolo161(&net, classes, 3, g161_bin, 3, 1.05); + + + + + + + yolo[0] = &yolo139; + yolo[1] = &yolo150; + yolo[2] = &yolo161; + + // fill classes names + for (int i = 0; i < 3; i++) + { + yolo[i]->classesNames = {"person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"}; + } + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + + //print network model + net.print(); + + // //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo4_608")); + + // the network have 3 outputs + tk::dnn::dataDim_t out_dim[3]; + for (int i = 0; i < 3; i++) + out_dim[i] = yolo[i]->output_dim; + dnnType *cudnn_out[3], *rt_out[3]; + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + + for (int i = 0; i < 3; i++) + cudnn_out[i] = yolo[i]->dstData; + + printCenteredTitle(" compute detections ", '=', 30); + TIMER_START + int ndets = 0; + tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); + for (int i = 0; i < 3; i++) + yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); + tk::dnn::Yolo::mergeDetections(dets, ndets, classes); + + for (int j = 0; j < ndets; j++) + { + tk::dnn::Yolo::box b = dets[j].bbox; + int x0 = (b.x - b.w / 2.); + int x1 = (b.x + b.w / 2.); + int y0 = (b.y - b.h / 2.); + int y1 = (b.y + b.h / 2.); + + int cl = 0; + for (int c = 0; c < classes; ++c) + { + float prob = dets[j].prob[c]; + if (prob > 0) + cl = c; + } + std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; + } + TIMER_STOP + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + for (int i = 0; i < 3; i++) + rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; + + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for (int i = 0; i < 3; i++) + { + printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); + dnnType *out, *out_h; + int odim = out_dim[i].tot(); + readBinaryFile(output_bins[i], odim, &out_h, &out); + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} -- 2.52.0 From 2e3cb52cff46f5990e88aa3223e39e96df4c8702 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 11 May 2020 15:08:10 +0200 Subject: [PATCH 002/228] Add flag to disable visualization and save video Signed-off-by: Micaela Verucchi --- demo/demo/demo.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 75012b6..540ee25 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -34,6 +34,12 @@ int main(int argc, char *argv[]) { int n_classes = 80; if(argc > 4) n_classes = atoi(argv[4]); + bool show = true; + if(argc > 5) + show = atoi(argv[5]); + + if(!show) + SAVE_RESULT = true; tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; @@ -76,7 +82,8 @@ int main(int argc, char *argv[]) { cv::Mat frame; cv::Mat dnn_input; - cv::namedWindow("detection", cv::WINDOW_NORMAL); + if(show) + cv::namedWindow("detection", cv::WINDOW_NORMAL); std::vector detected_bbox; @@ -93,8 +100,10 @@ int main(int argc, char *argv[]) { detNN->update(dnn_input); frame = detNN->draw(frame); - cv::imshow("detection", frame); - cv::waitKey(1); + if(show){ + cv::imshow("detection", frame); + cv::waitKey(1); + } if(SAVE_RESULT) resultVideo << frame; } -- 2.52.0 From 40456592fc6693ca224202a88d78bbaa20446cce Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 14 May 2020 16:41:51 +0200 Subject: [PATCH 003/228] Adapt detection classes to use batches, adapt demos, update README Signed-off-by: Micaela Verucchi --- README.md | 7 ++- demo/demo/demo.cpp | 58 ++++++++++++++--------- demo/demo/map.cpp | 17 +++---- include/tkDNN/CenternetDetection.h | 6 +-- include/tkDNN/DetectionNN.h | 75 ++++++++++++++++++------------ include/tkDNN/MobilenetDetection.h | 6 +-- include/tkDNN/Yolo3Detection.h | 6 +-- src/CenternetDetection.cpp | 26 ++++++----- src/MobilenetDetection.cpp | 22 +++++---- src/Yolo3Detection.cpp | 29 +++++++----- 10 files changed, 149 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 7f1016d..0593bcb 100644 --- a/README.md +++ b/README.md @@ -123,13 +123,16 @@ rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files ``` In general the demo program takes 4 parameters: ``` -./demo +./demo ``` where * `````` is the rt file generated by a test * ```<``` is the path to a video file or a camera input * `````` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family) * ``````is the number of classes the network is trained on +* `````` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network). +* `````` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1) + N.b. By default it is used FP32 inference ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) @@ -218,6 +221,8 @@ cd build ./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml ``` +This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to subit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate). + ## Existing tests and supported networks | Test Name | Network | Dataset | N Classes | Input size | Weights | diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 540ee25..67d5786 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -34,9 +34,15 @@ int main(int argc, char *argv[]) { int n_classes = 80; if(argc > 4) n_classes = atoi(argv[4]); - bool show = true; + int n_batch = 1; if(argc > 5) - show = atoi(argv[5]); + n_batch = atoi(argv[5]); + bool show = true; + if(argc > 6) + show = atoi(argv[6]); + + if(n_batch < 1 || n_batch > 64) + FatalError("Batch dim not supported"); if(!show) SAVE_RESULT = true; @@ -63,7 +69,7 @@ int main(int argc, char *argv[]) { FatalError("Network type not allowed (3rd parameter)\n"); } - detNN->init(net, n_classes); + detNN->init(net, n_classes, n_batch); gRun = true; @@ -81,30 +87,40 @@ int main(int argc, char *argv[]) { } cv::Mat frame; - cv::Mat dnn_input; if(show) cv::namedWindow("detection", cv::WINDOW_NORMAL); - - std::vector detected_bbox; + + std::vector batch_frame; + std::vector batch_dnn_input; while(gRun) { - cap >> frame; - if(!frame.data) { - break; - } - - // this will be resized to the net format - dnn_input = frame.clone(); + batch_dnn_input.clear(); + batch_frame.clear(); + for(int bi=0; bi< n_batch; ++bi){ + cap >> frame; + if(!frame.data) + break; + + batch_frame.push_back(frame); + + // this will be resized to the net format + batch_dnn_input.push_back(frame.clone()); + } + if(!frame.data) + break; + //inference - detNN->update(dnn_input); - frame = detNN->draw(frame); + detNN->update(batch_dnn_input); + detNN->draw(batch_frame); if(show){ - cv::imshow("detection", frame); - cv::waitKey(1); + for(int bi=0; bi< n_batch; ++bi){ + cv::imshow("detection", batch_frame[bi]); + cv::waitKey(1); + } } - if(SAVE_RESULT) + if(n_batch == 1 && SAVE_RESULT) resultVideo << frame; } @@ -112,10 +128,10 @@ int main(int argc, char *argv[]) { double mean = 0; std::cout<stats.begin(), detNN->stats.end())<<" ms\n"; - std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n"; + std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); - std::cout<<"Avg: "< batch_frames; + batch_frames.push_back(frame); int height = frame.rows; int width = frame.cols; - cv::Mat dnn_input; if(!frame.data) break; - dnn_input = frame.clone(); + std::vector batch_dnn_input; + batch_dnn_input.push_back(frame.clone()); //inference - detected_bbox.clear(); - detNN->update(dnn_input, write_res_on_file, ×, write_coco_json); - frame = detNN->draw(frame); + detNN->update(batch_dnn_input, write_res_on_file, ×, write_coco_json); + detNN->draw(batch_frames); detected_bbox = detNN->detected; if(write_coco_json) @@ -171,7 +172,7 @@ int main(int argc, char *argv[]) myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; if(show)// draw rectangle for detection - cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); + cv::rectangle(batch_frames[0], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); } if(write_dets) @@ -190,14 +191,14 @@ int main(int argc, char *argv[]) f.gt.push_back(b); if(show)// draw rectangle for groundtruth - cv::rectangle(frame, cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2); + cv::rectangle(batch_frames[0], cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2); } } images.push_back(f); if(show){ - cv::imshow("detection", frame); + cv::imshow("detection", batch_frames[0]); cv::waitKey(0); } diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 92feba5..227cb78 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -73,9 +73,9 @@ public: CenternetDetection() {}; ~CenternetDetection() {}; - bool init(const std::string& tensor_path, const int n_classes=80); - void preprocess(cv::Mat &frame); - void postprocess(const bool mAP=false); + bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1); + void preprocess(cv::Mat &frame, const int bi=0); + void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index a635b2d..2948111 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -34,6 +34,8 @@ class DetectionNN { cv::Scalar colors[256]; + int nBatches = 1; + #ifdef OPENCV_CUDACONTRIB cv::cuda::GpuMat bgr[3]; cv::cuda::GpuMat imagePreproc; @@ -47,21 +49,26 @@ class DetectionNN { * This method preprocess the image, before feeding it to the NN. * * @param frame original frame to adapt for inference. + * @param bi batch index */ - virtual void preprocess(cv::Mat &frame) = 0; + virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; /** * This method postprocess the output of the NN to obtain the correct * boundig boxes. * + * @param bi batch index + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation */ - virtual void postprocess(const bool mAP=false) = 0; + virtual void postprocess(const int bi=0,const bool mAP=false) = 0; public: int classes = 0; - float confThreshold = 0.05; /*threshold on the confidence of the boxes*/ + float confThreshold = 0.3; /*threshold on the confidence of the boxes*/ std::vector detected; /*bounding boxes in output*/ + std::vector> batchDetected; /*bounding boxes in output*/ std::vector stats; /*keeps track of inference times (ms)*/ std::vector classesNames; @@ -74,36 +81,41 @@ class DetectionNN { * * @param tensor_path path to the rt file og the NN. * @param n_classes number of classes for the given dataset. + * @param n_batches number of batches to use in inference * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path, const int n_classes=80) = 0; + virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1) = 0; /** * This method performs the whole detection of the NN. * - * @param frame frame to run detection on. + * @param frames frames to run detection on. * @param save_times if set to true, preprocess, inference and postprocess times * are saved on a csv file, otherwise not. * @param times pointer to the output stream where to write times + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation */ - void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ - if(!frame.data) - FatalError("No image data feed to detection"); - + void update(std::vector& frames, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ if(save_times && times==nullptr) FatalError("save_times set to true, but no valid ofstream given"); - originalSize = frame.size(); printCenteredTitle(" TENSORRT detection ", '=', 30); { TIMER_START - preprocess(frame); + for(int bi=0; biinput_dim; + dim.n = nBatches; { dim.print(); TIMER_START @@ -114,9 +126,11 @@ class DetectionNN { if(save_times) *times<& frames) { tk::dnn::box b; int x0, w, x1, y0, h, y1; int objClass; @@ -137,24 +150,26 @@ class DetectionNN { int baseline = 0; float font_scale = 0.5; int thickness = 2; - // draw dets - for(int i=0; iinput_dim; @@ -41,7 +42,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe trans = cv::Mat(cv::Size(3,2), CV_32F); trans2 = cv::Mat(cv::Size(3,2), CV_32F); - checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); dim_hm = tk::dnn::dataDim_t(1, 80, 128, 128, 1); dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); @@ -98,7 +99,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); #else - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot()* nBatches)); mean << 0.408, 0.447, 0.47; stddev << 0.289, 0.274, 0.278; #endif @@ -120,7 +121,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe } -void CenternetDetection::preprocess(cv::Mat &frame){ +void CenternetDetection::preprocess(cv::Mat &frame, const int bi){ // -----------------------------------pre-process ------------------------------------------ // auto start_t = std::chrono::steady_clock::now(); @@ -212,7 +213,7 @@ void CenternetDetection::preprocess(cv::Mat &frame){ // std::cout << " TIME normalize: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; // step_t = end_t; - checkCuda(cudaMemcpy(input_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); // end_t = std::chrono::steady_clock::now(); // std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; @@ -254,18 +255,18 @@ void CenternetDetection::preprocess(cv::Mat &frame){ int idx = i*imageF.rows*imageF.cols; int ch = dim2.c-3 +i; // std::cout<<"i: "<input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType)); } - checkCuda(cudaMemcpyAsync(input_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice)); + checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice)); #endif } -void CenternetDetection::postprocess(const bool mAP){ +void CenternetDetection::postprocess(const int bi, const bool mAP){ dnnType *rt_out[4]; - rt_out[0] = (dnnType *)netRT->buffersRT[1]; - rt_out[1] = (dnnType *)netRT->buffersRT[2]; - rt_out[2] = (dnnType *)netRT->buffersRT[3]; - rt_out[3] = (dnnType *)netRT->buffersRT[4]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[0].tot()*bi; + rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[1].tot()*bi; + rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[2].tot()*bi; + rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[3].tot()*bi; // auto start_t = std::chrono::steady_clock::now(); // auto step_t = std::chrono::steady_clock::now(); @@ -389,6 +390,7 @@ void CenternetDetection::postprocess(const bool mAP){ } } + batchDetected.push_back(detected); // end_t = std::chrono::steady_clock::now(); // std::cout << " TIME detections: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; // step_t = end_t; diff --git a/src/MobilenetDetection.cpp b/src/MobilenetDetection.cpp index d66a8cc..4289d96 100644 --- a/src/MobilenetDetection.cpp +++ b/src/MobilenetDetection.cpp @@ -126,11 +126,12 @@ float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){ return iou; } -bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes){ +bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); imageSize = netRT->input_dim.h; classes = n_classes; + nBatches = n_batches; SSDSpec specs[N_SSDSPEC]; @@ -157,9 +158,9 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe generate_ssd_priors(specs, N_SSDSPEC); #ifndef OPENCV_CUDACONTRIB - checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot())); + checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); #endif - checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); locations_h = (float *)malloc(N_COORDS * nPriors * sizeof(float)); confidences_h = (float *)malloc(nPriors * classes * sizeof(float)); @@ -208,7 +209,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe return 1; } -void MobilenetDetection::preprocess(cv::Mat &frame){ +void MobilenetDetection::preprocess(cv::Mat &frame, const int bi){ #ifdef OPENCV_CUDACONTRIB //move original image on GPU cv::cuda::GpuMat orig_img, frame_nomean; @@ -224,7 +225,7 @@ void MobilenetDetection::preprocess(cv::Mat &frame){ for(int i=0; i < netRT->input_dim.c; i++){ int idx = i * imagePreproc.rows * imagePreproc.cols; - checkCuda( cudaMemcpy((void *)&input_d[idx], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols* sizeof(float), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaMemcpy((void *)&input_d[idx + netRT->input_dim.tot()*bi], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols* sizeof(float), cudaMemcpyDeviceToDevice) ); } #else //resize image, remove mean, divide by std @@ -237,17 +238,17 @@ void MobilenetDetection::preprocess(cv::Mat &frame){ cv::split(imagePreproc, bgr); for (int i = 0; i < netRT->input_dim.c; i++){ int idx = i * imagePreproc.rows * imagePreproc.cols; - memcpy((void *)&input[idx], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType)); + memcpy((void *)&input[idx + netRT->input_dim.tot()*bi], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType)); } - checkCuda(cudaMemcpyAsync(input_d, input, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); + checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); #endif } -void MobilenetDetection::postprocess(const bool mAP){ +void MobilenetDetection::postprocess(const int bi, const bool mAP){ //get confidences and locations_h dnnType *rt_out[2]; - rt_out[0] = (dnnType *)netRT->buffersRT[3]; - rt_out[1] = (dnnType *)netRT->buffersRT[4]; + rt_out[0] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi; + rt_out[1] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi; detected.clear(); @@ -302,6 +303,7 @@ void MobilenetDetection::postprocess(const bool mAP){ boxes = remaining; } } + batchDetected.push_back(detected); } diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index 18173f4..fc5e1d0 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -3,12 +3,16 @@ namespace tk { namespace dnn { -bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes) { +bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches) { //convert network to tensorRT std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); + nBatches = n_batches; + tk::dnn::dataDim_t idim = netRT->input_dim; + idim.n = nBatches; + if(netRT->pluginFactory->n_yolos < 2 ) { FatalError("this is not yolo3"); } @@ -19,7 +23,7 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes) { num = yRT->num; nMasks = yRT->n_masks; - // make a yolo layer for interpret predictions + // make a yolo layer to interpret predictions yolo[i] = new tk::dnn::Yolo(nullptr, classes, nMasks, ""); // yolo without input and bias yolo[i]->mask_h = new dnnType[nMasks]; yolo[i]->bias_h = new dnnType[num*nMasks*2]; @@ -31,9 +35,9 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes) { dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); #ifndef OPENCV_CUDACONTRIB - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*idim.tot())); #endif - checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*idim.tot())); // class colors precompute for(int c=0; cinput_dim.c-1 -i; bgr[ch].download(bgr_h); //TODO: don't copy back on CPU - checkCuda( cudaMemcpy(input_d + i*size, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice)); + checkCuda( cudaMemcpy(input_d + i*size + netRT->input_dim.tot()*bi, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice)); } #else cv::resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); @@ -77,18 +81,18 @@ void Yolo3Detection::preprocess(cv::Mat &frame){ for(int i=0; iinput_dim.c; i++) { int idx = i*imagePreproc.rows*imagePreproc.cols; int ch = netRT->input_dim.c-1 -i; - memcpy((void*)&input[idx], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType)); + memcpy((void*)&input[idx + netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType)); } - checkCuda(cudaMemcpyAsync(input_d, input, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); + checkCuda(cudaMemcpyAsync(input_d + netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); #endif } -void Yolo3Detection::postprocess(const bool mAP){ +void Yolo3Detection::postprocess(const int bi, const bool mAP){ + //get yolo outputs dnnType *rt_out[netRT->pluginFactory->n_yolos]; - for(int i=0; ipluginFactory->n_yolos; i++) { - rt_out[i] = (dnnType*)netRT->buffersRT[i+1]; - } + for(int i=0; ipluginFactory->n_yolos; i++) + rt_out[i] = (dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi; float x_ratio = float(originalSize.width) / float(netRT->input_dim.w); float y_ratio = float(originalSize.height) / float(netRT->input_dim.h); @@ -138,6 +142,7 @@ void Yolo3Detection::postprocess(const bool mAP){ detected.push_back(res); } } + batchDetected.push_back(detected); } -- 2.52.0 From 5d01a3f6295cadc26090d4ab95d0193b31d94fcf Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 14 May 2020 17:13:38 +0200 Subject: [PATCH 004/228] Fix Centernet postprocessing Signed-off-by: Micaela Verucchi --- src/CenternetDetection.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index cef1375..5d01cc3 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -263,10 +263,10 @@ void CenternetDetection::preprocess(cv::Mat &frame, const int bi){ void CenternetDetection::postprocess(const int bi, const bool mAP){ dnnType *rt_out[4]; - rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[0].tot()*bi; - rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[1].tot()*bi; - rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[2].tot()*bi; - rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[3].tot()*bi; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi; + rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi; + rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi; // auto start_t = std::chrono::steady_clock::now(); // auto step_t = std::chrono::steady_clock::now(); -- 2.52.0 From 23b40de508f7b2e9ae79dab0269cb3a3f4734681 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Fri, 15 May 2020 09:41:27 +0200 Subject: [PATCH 005/228] Add verbose define Signed-off-by: Micaela Verucchi --- demo/demo/demo.cpp | 2 +- include/tkDNN/DetectionNN.h | 6 +++--- include/tkDNN/utils.h | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 67d5786..afa5577 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -131,7 +131,7 @@ int main(int argc, char *argv[]) { std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); - std::cout<<"Avg: "<input_dim; dim.n = nBatches; { - dim.print(); + if(VERBOSE) dim.print(); TIMER_START netRT->infer(dim, input_d); TIMER_STOP - dim.print(); + if(VERBOSE) dim.print(); stats.push_back(t_ns); if(save_times) *times< Date: Fri, 15 May 2020 09:48:59 +0200 Subject: [PATCH 006/228] Add yolo4_berkeley test Signed-off-by: Micaela Verucchi --- tests/yolo4_berkeley/yolo4_berkeley.cfg | 1159 +++++++++++++++++++++++ tests/yolo4_berkeley/yolo4_berkeley.cpp | 666 +++++++++++++ 2 files changed, 1825 insertions(+) create mode 100644 tests/yolo4_berkeley/yolo4_berkeley.cfg create mode 100644 tests/yolo4_berkeley/yolo4_berkeley.cpp diff --git a/tests/yolo4_berkeley/yolo4_berkeley.cfg b/tests/yolo4_berkeley/yolo4_berkeley.cfg new file mode 100644 index 0000000..b11c0a7 --- /dev/null +++ b/tests/yolo4_berkeley/yolo4_berkeley.cfg @@ -0,0 +1,1159 @@ +[net] +# Testing +#batch=1 +#subdivisions=1 +# Training +batch=64 +subdivisions=16 +width=544 +height=320 +channels=3 +momentum=0.949 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=1000 +max_batches = 20000 +policy=steps +steps=16000d,18000 +scales=.1,.1 + +#cutmix=1 +mosaic=1 + +#:104x104 54:52x52 85:26x26 104:13x13 for 416 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-7 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-10 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-16 + +[convolutional] +batch_normalize=1 +filters=1024 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +### SPP ### +[maxpool] +stride=1 +size=5 + +[route] +layers=-2 + +[maxpool] +stride=1 +size=9 + +[route] +layers=-4 + +[maxpool] +stride=1 +size=13 + +[route] +layers=-1,-3,-5,-6 +### End SPP ### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = 85 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = 54 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +########################## + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=45 +activation=linear + + +[yolo] +mask = 0,1,2 +anchors = 6, 7, 14, 11, 9, 19, 26, 20, 18, 42, 48, 35, 74, 65, 126, 99, 183,169 +classes=10 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +scale_x_y = 1.2 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=256 +activation=leaky + +[route] +layers = -1, -16 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=45 +activation=linear + + +[yolo] +mask = 3,4,5 +anchors = 6, 7, 14, 11, 9, 19, 26, 20, 18, 42, 48, 35, 74, 65, 126, 99, 183,169 +classes=10 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +scale_x_y = 1.1 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=512 +activation=leaky + +[route] +layers = -1, -37 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=45 +activation=linear + + +[yolo] +mask = 6,7,8 +anchors = 6, 7, 14, 11, 9, 19, 26, 20, 18, 42, 48, 35, 74, 65, 126, 99, 183,169 +classes=10 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 +scale_x_y = 1.05 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + diff --git a/tests/yolo4_berkeley/yolo4_berkeley.cpp b/tests/yolo4_berkeley/yolo4_berkeley.cpp new file mode 100644 index 0000000..d10ad59 --- /dev/null +++ b/tests/yolo4_berkeley/yolo4_berkeley.cpp @@ -0,0 +1,666 @@ +#include +#include +#include "tkdnn.h" + +int main() +{ + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 320, 544, 1); + tk::dnn::Network net(dim); + + // create yolo4_berkeley model + std::string bin_path = "yolo4_berkeley"; + int classes = 10; + tk::dnn::Yolo *yolo[3]; + + std::string input_bin = bin_path + "/layers/input.bin"; + + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin"}; + std::string c0_bin = bin_path + "/layers/c0.bin"; + std::string c1_bin = bin_path + "/layers/c1.bin"; + std::string c2_bin = bin_path + "/layers/c2.bin"; + std::string c3_bin = bin_path + "/layers/c3.bin"; + std::string c4_bin = bin_path + "/layers/c4.bin"; + std::string c5_bin = bin_path + "/layers/c5.bin"; + std::string c6_bin = bin_path + "/layers/c6.bin"; + std::string c7_bin = bin_path + "/layers/c7.bin"; + std::string c8_bin = bin_path + "/layers/c8.bin"; + std::string c10_bin = bin_path + "/layers/c10.bin"; + std::string c11_bin = bin_path + "/layers/c11.bin"; + std::string c12_bin = bin_path + "/layers/c12.bin"; + std::string c13_bin = bin_path + "/layers/c13.bin"; + std::string c14_bin = bin_path + "/layers/c14.bin"; + std::string c15_bin = bin_path + "/layers/c15.bin"; + std::string c16_bin = bin_path + "/layers/c16.bin"; + std::string c17_bin = bin_path + "/layers/c17.bin"; + std::string c18_bin = bin_path + "/layers/c18.bin"; + std::string c19_bin = bin_path + "/layers/c19.bin"; + std::string c20_bin = bin_path + "/layers/c20.bin"; + std::string c21_bin = bin_path + "/layers/c21.bin"; + std::string c23_bin = bin_path + "/layers/c23.bin"; + std::string c24_bin = bin_path + "/layers/c24.bin"; + std::string c25_bin = bin_path + "/layers/c25.bin"; + std::string c26_bin = bin_path + "/layers/c26.bin"; + std::string c27_bin = bin_path + "/layers/c27.bin"; + std::string c28_bin = bin_path + "/layers/c28.bin"; + std::string c29_bin = bin_path + "/layers/c29.bin"; + std::string c30_bin = bin_path + "/layers/c30.bin"; + std::string c31_bin = bin_path + "/layers/c31.bin"; + std::string c32_bin = bin_path + "/layers/c32.bin"; + std::string c33_bin = bin_path + "/layers/c33.bin"; + std::string c34_bin = bin_path + "/layers/c34.bin"; + std::string c35_bin = bin_path + "/layers/c35.bin"; + std::string c36_bin = bin_path + "/layers/c36.bin"; + std::string c37_bin = bin_path + "/layers/c37.bin"; + std::string c38_bin = bin_path + "/layers/c38.bin"; + std::string c39_bin = bin_path + "/layers/c39.bin"; + std::string c40_bin = bin_path + "/layers/c40.bin"; + std::string c41_bin = bin_path + "/layers/c41.bin"; + std::string c42_bin = bin_path + "/layers/c42.bin"; + std::string c43_bin = bin_path + "/layers/c43.bin"; + std::string c44_bin = bin_path + "/layers/c44.bin"; + std::string c45_bin = bin_path + "/layers/c45.bin"; + std::string c46_bin = bin_path + "/layers/c46.bin"; + std::string c47_bin = bin_path + "/layers/c47.bin"; + std::string c48_bin = bin_path + "/layers/c48.bin"; + std::string c49_bin = bin_path + "/layers/c49.bin"; + std::string c50_bin = bin_path + "/layers/c50.bin"; + std::string c51_bin = bin_path + "/layers/c51.bin"; + std::string c52_bin = bin_path + "/layers/c52.bin"; + std::string c53_bin = bin_path + "/layers/c53.bin"; + std::string c54_bin = bin_path + "/layers/c54.bin"; + std::string c55_bin = bin_path + "/layers/c55.bin"; + std::string c56_bin = bin_path + "/layers/c56.bin"; + std::string c57_bin = bin_path + "/layers/c57.bin"; + std::string c58_bin = bin_path + "/layers/c58.bin"; + std::string c59_bin = bin_path + "/layers/c59.bin"; + std::string c60_bin = bin_path + "/layers/c60.bin"; + std::string c61_bin = bin_path + "/layers/c61.bin"; + std::string c62_bin = bin_path + "/layers/c62.bin"; + std::string c63_bin = bin_path + "/layers/c63.bin"; + std::string c65_bin = bin_path + "/layers/c65.bin"; + std::string c66_bin = bin_path + "/layers/c66.bin"; + std::string c67_bin = bin_path + "/layers/c67.bin"; + std::string c68_bin = bin_path + "/layers/c68.bin"; + std::string c69_bin = bin_path + "/layers/c69.bin"; + std::string c70_bin = bin_path + "/layers/c70.bin"; + std::string c71_bin = bin_path + "/layers/c71.bin"; + std::string c72_bin = bin_path + "/layers/c72.bin"; + std::string c74_bin = bin_path + "/layers/c74.bin"; + std::string c75_bin = bin_path + "/layers/c75.bin"; + std::string c76_bin = bin_path + "/layers/c76.bin"; + std::string c77_bin = bin_path + "/layers/c77.bin"; + std::string c78_bin = bin_path + "/layers/c78.bin"; + std::string c80_bin = bin_path + "/layers/c80.bin"; + std::string c81_bin = bin_path + "/layers/c81.bin"; + std::string c82_bin = bin_path + "/layers/c82.bin"; + std::string c83_bin = bin_path + "/layers/c83.bin"; + std::string c85_bin = bin_path + "/layers/c85.bin"; + std::string c86_bin = bin_path + "/layers/c86.bin"; + std::string c87_bin = bin_path + "/layers/c87.bin"; + std::string c89_bin = bin_path + "/layers/c89.bin"; + std::string c90_bin = bin_path + "/layers/c90.bin"; + std::string c91_bin = bin_path + "/layers/c91.bin"; + std::string c92_bin = bin_path + "/layers/c92.bin"; + std::string c93_bin = bin_path + "/layers/c93.bin"; + std::string c94_bin = bin_path + "/layers/c94.bin"; + std::string c96_bin = bin_path + "/layers/c96.bin"; + std::string c97_bin = bin_path + "/layers/c97.bin"; + std::string c98_bin = bin_path + "/layers/c98.bin"; + std::string c99_bin = bin_path + "/layers/c99.bin"; + std::string c100_bin = bin_path + "/layers/c100.bin"; + std::string c101_bin = bin_path + "/layers/c101.bin"; + std::string c102_bin = bin_path + "/layers/c102.bin"; + std::string c103_bin = bin_path + "/layers/c103.bin"; + std::string c104_bin = bin_path + "/layers/c104.bin"; + std::string c105_bin = bin_path + "/layers/c105.bin"; + std::string c106_bin = bin_path + "/layers/c106.bin"; + std::string c107_bin = bin_path + "/layers/c107.bin"; + std::string c108_bin = bin_path + "/layers/c108.bin"; + std::string c109_bin = bin_path + "/layers/c109.bin"; + std::string c110_bin = bin_path + "/layers/c110.bin"; + std::string c111_bin = bin_path + "/layers/c111.bin"; + std::string c112_bin = bin_path + "/layers/c112.bin"; + std::string c113_bin = bin_path + "/layers/c113.bin"; + std::string c114_bin = bin_path + "/layers/c114.bin"; + std::string c115_bin = bin_path + "/layers/c115.bin"; + std::string c116_bin = bin_path + "/layers/c116.bin"; + std::string c117_bin = bin_path + "/layers/c117.bin"; + std::string c119_bin = bin_path + "/layers/c119.bin"; + std::string c120_bin = bin_path + "/layers/c120.bin"; + std::string c121_bin = bin_path + "/layers/c121.bin"; + std::string c122_bin = bin_path + "/layers/c122.bin"; + std::string c123_bin = bin_path + "/layers/c123.bin"; + std::string c124_bin = bin_path + "/layers/c124.bin"; + std::string c125_bin = bin_path + "/layers/c125.bin"; + std::string c126_bin = bin_path + "/layers/c126.bin"; + std::string c127_bin = bin_path + "/layers/c127.bin"; + std::string c128_bin = bin_path + "/layers/c128.bin"; + std::string c130_bin = bin_path + "/layers/c130.bin"; + std::string c131_bin = bin_path + "/layers/c131.bin"; + std::string c132_bin = bin_path + "/layers/c132.bin"; + std::string c133_bin = bin_path + "/layers/c133.bin"; + std::string c134_bin = bin_path + "/layers/c134.bin"; + std::string c135_bin = bin_path + "/layers/c135.bin"; + std::string c136_bin = bin_path + "/layers/c136.bin"; + std::string c137_bin = bin_path + "/layers/c137.bin"; + std::string c138_bin = bin_path + "/layers/c138.bin"; + std::string c141_bin = bin_path + "/layers/c141.bin"; + std::string c142_bin = bin_path + "/layers/c142.bin"; + std::string c143_bin = bin_path + "/layers/c143.bin"; + std::string c144_bin = bin_path + "/layers/c144.bin"; + std::string c145_bin = bin_path + "/layers/c145.bin"; + std::string c146_bin = bin_path + "/layers/c146.bin"; + std::string c147_bin = bin_path + "/layers/c147.bin"; + std::string c148_bin = bin_path + "/layers/c148.bin"; + std::string c149_bin = bin_path + "/layers/c149.bin"; + std::string c150_bin = bin_path + "/layers/c150.bin"; + std::string c151_bin = bin_path + "/layers/c151.bin"; + std::string c152_bin = bin_path + "/layers/c152.bin"; + std::string c153_bin = bin_path + "/layers/c153.bin"; + std::string c154_bin = bin_path + "/layers/c154.bin"; + std::string c155_bin = bin_path + "/layers/c155.bin"; + std::string c156_bin = bin_path + "/layers/c156.bin"; + std::string c157_bin = bin_path + "/layers/c157.bin"; + std::string c158_bin = bin_path + "/layers/c158.bin"; + std::string c159_bin = bin_path + "/layers/c159.bin"; + std::string c160_bin = bin_path + "/layers/c160.bin"; + std::string g139_bin = bin_path + "/layers/g139.bin"; + std::string g150_bin = bin_path + "/layers/g150.bin"; + std::string g161_bin = bin_path + "/layers/g161.bin"; + + + downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); + + tk::dnn::Conv2d c0(&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); + tk::dnn::Activation a0(&net, tk::dnn::ACTIVATION_MISH); + + // downsample + tk::dnn::Conv2d c1(&net, 64, 3, 3, 2, 2, 1, 1, c1_bin, true); + tk::dnn::Activation a1(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c2(&net, 64, 1, 1, 1, 1, 0, 0, c2_bin, true); + tk::dnn::Activation a2(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r3_layers[1] = {&a1}; + tk::dnn::Route r3(&net, r3_layers, 1); + + tk::dnn::Conv2d c4(&net, 64, 1, 1, 1, 1, 0, 0, c4_bin, true); + tk::dnn::Activation a4(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c5(&net, 32, 1, 1, 1, 1, 0, 0, c5_bin, true); + tk::dnn::Activation a5(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c6(&net, 64, 3, 3, 1, 1, 1, 1, c6_bin, true); + tk::dnn::Activation a6(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s7(&net, &a4); + + tk::dnn::Conv2d c8(&net, 64, 1, 1, 1, 1, 0, 0, c8_bin, true); + tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r9_layers[2] = {&a8, &a2}; + tk::dnn::Route r9(&net, r9_layers, 2); + + tk::dnn::Conv2d c10(&net, 64, 1, 1, 1, 1, 0, 0, c10_bin, true); + tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_MISH); + + // downsample + tk::dnn::Conv2d c11(&net, 128, 3, 3, 2, 2, 1, 1, c11_bin, true); + tk::dnn::Activation a11(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c12(&net, 64, 1, 1, 1, 1, 0, 0, c12_bin, true); + tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r13_layers[1] = {&a11}; + tk::dnn::Route r13(&net, r13_layers, 1); + + tk::dnn::Conv2d c14(&net, 64, 1, 1, 1, 1, 0, 0, c14_bin, true); + tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c15(&net, 64, 1, 1, 1, 1, 0, 0, c15_bin, true); + tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c16(&net, 64, 3, 3, 1, 1, 1, 1, c16_bin, true); + tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s17(&net, &a14); + + tk::dnn::Conv2d c18(&net, 64, 1, 1, 1, 1, 0, 0, c18_bin, true); + tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c19(&net, 64, 3, 3, 1, 1, 1, 1, c19_bin, true); + tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s20(&net, &s17); + + tk::dnn::Conv2d c21(&net, 64, 1, 1, 1, 1, 0, 0, c21_bin, true); + tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r22_layers[2] = {&a21, &a12}; + tk::dnn::Route r22(&net, r22_layers, 2); + + tk::dnn::Conv2d c23(&net, 128, 1, 1, 1, 1, 0, 0, c23_bin, true); + tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_MISH); + + //downsample + tk::dnn::Conv2d c24(&net, 256, 3, 3, 2, 2, 1, 1, c24_bin, true); + tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c25(&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true); + tk::dnn::Activation a25(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r26_layers[1] = {&a24}; + tk::dnn::Route r26(&net, r26_layers, 1); + + tk::dnn::Conv2d c27(&net, 128, 1, 1, 1, 1, 0, 0, c27_bin, true); + tk::dnn::Activation a27(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c28(&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true); + tk::dnn::Activation a28(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c29(&net, 128, 3, 3, 1, 1, 1, 1, c29_bin, true); + tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s30(&net, &a27); + + tk::dnn::Conv2d c31(&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true); + tk::dnn::Activation a31(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c32(&net, 128, 3, 3, 1, 1, 1, 1, c32_bin, true); + tk::dnn::Activation a32(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s33(&net, &s30); + + tk::dnn::Conv2d c34(&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true); + tk::dnn::Activation a34(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c35(&net, 128, 3, 3, 1, 1, 1, 1, c35_bin, true); + tk::dnn::Activation a35(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s36(&net, &s33); + + tk::dnn::Conv2d c37(&net, 128, 1, 1, 1, 1, 0, 0, c37_bin, true); + tk::dnn::Activation a37(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c38(&net, 128, 3, 3, 1, 1, 1, 1, c38_bin, true); + tk::dnn::Activation a38(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s39(&net, &s36); + + tk::dnn::Conv2d c40(&net, 128, 1, 1, 1, 1, 0, 0, c40_bin, true); + tk::dnn::Activation a40(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c41(&net, 128, 3, 3, 1, 1, 1, 1, c41_bin, true); + tk::dnn::Activation a41(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s42(&net, &s39); + + tk::dnn::Conv2d c43(&net, 128, 1, 1, 1, 1, 0, 0, c43_bin, true); + tk::dnn::Activation a43(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c44(&net, 128, 3, 3, 1, 1, 1, 1, c44_bin, true); + tk::dnn::Activation a44(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s45(&net, &s42); + + tk::dnn::Conv2d c46(&net, 128, 1, 1, 1, 1, 0, 0, c46_bin, true); + tk::dnn::Activation a46(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c47(&net, 128, 3, 3, 1, 1, 1, 1, c47_bin, true); + tk::dnn::Activation a47(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s48(&net, &s45); + + tk::dnn::Conv2d c49(&net, 128, 1, 1, 1, 1, 0, 0, c49_bin, true); + tk::dnn::Activation a49(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c50(&net, 128, 3, 3, 1, 1, 1, 1, c50_bin, true); + tk::dnn::Activation a50(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s51(&net, &s48); + + tk::dnn::Conv2d c52(&net, 128, 1, 1, 1, 1, 0, 0, c52_bin, true); + tk::dnn::Activation a52(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r53_layers[2] = {&a52, &a25}; + tk::dnn::Route r53(&net, r53_layers, 2); + + tk::dnn::Conv2d c54(&net, 256, 1, 1, 1, 1, 0, 0, c54_bin, true); + tk::dnn::Activation a54(&net, tk::dnn::ACTIVATION_MISH); + + //downsample + tk::dnn::Conv2d c55(&net, 512, 3, 3, 2, 2, 1, 1, c55_bin, true); + tk::dnn::Activation a55(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c56(&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true); + tk::dnn::Activation a56(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r57_layers[1] = {&a55}; + tk::dnn::Route r57(&net, r57_layers, 1); + + tk::dnn::Conv2d c58(&net, 256, 1, 1, 1, 1, 0, 0, c58_bin, true); + tk::dnn::Activation a58(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c59(&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true); + tk::dnn::Activation a59(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c60(&net, 256, 3, 3, 1, 1, 1, 1, c60_bin, true); + tk::dnn::Activation a60(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s61(&net, &a58); + + tk::dnn::Conv2d c62(&net, 256, 1, 1, 1, 1, 0, 0, c62_bin, true); + tk::dnn::Activation a62(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c63(&net, 256, 3, 3, 1, 1, 1, 1, c63_bin, true); + tk::dnn::Activation a63(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s64(&net, &s61); + + tk::dnn::Conv2d c65(&net, 256, 1, 1, 1, 1, 0, 0, c65_bin, true); + tk::dnn::Activation a65(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c66(&net, 256, 3, 3, 1, 1, 1, 1, c66_bin, true); + tk::dnn::Activation a66(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s67(&net, &s64); + + tk::dnn::Conv2d c68(&net, 256, 1, 1, 1, 1, 0, 0, c68_bin, true); + tk::dnn::Activation a68(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c69(&net, 256, 3, 3, 1, 1, 1, 1, c69_bin, true); + tk::dnn::Activation a69(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s70(&net, &s67); + + tk::dnn::Conv2d c71(&net, 256, 1, 1, 1, 1, 0, 0, c71_bin, true); + tk::dnn::Activation a71(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c72(&net, 256, 3, 3, 1, 1, 1, 1, c72_bin, true); + tk::dnn::Activation a72(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s73(&net, &s70); + + tk::dnn::Conv2d c74(&net, 256, 1, 1, 1, 1, 0, 0, c74_bin, true); + tk::dnn::Activation a74(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c75(&net, 256, 3, 3, 1, 1, 1, 1, c75_bin, true); + tk::dnn::Activation a75(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s76(&net, &s73); + + tk::dnn::Conv2d c77(&net, 256, 1, 1, 1, 1, 0, 0, c77_bin, true); + tk::dnn::Activation a77(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c78(&net, 256, 3, 3, 1, 1, 1, 1, c78_bin, true); + tk::dnn::Activation a78(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s79(&net, &s76); + + tk::dnn::Conv2d c80(&net, 256, 1, 1, 1, 1, 0, 0, c80_bin, true); + tk::dnn::Activation a80(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c81(&net, 256, 3, 3, 1, 1, 1, 1, c81_bin, true); + tk::dnn::Activation a81(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s82(&net, &s79); + + tk::dnn::Conv2d c83(&net, 256, 1, 1, 1, 1, 0, 0, c83_bin, true); + tk::dnn::Activation a83(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r84_layers[2] = {&a83, &a56}; + tk::dnn::Route r84(&net, r84_layers, 2); + + tk::dnn::Conv2d c85(&net, 512, 1, 1, 1, 1, 0, 0, c85_bin, true); + tk::dnn::Activation a85(&net, tk::dnn::ACTIVATION_MISH); + + //downsample + tk::dnn::Conv2d c86(&net, 1024, 3, 3, 2, 2, 1, 1, c86_bin, true); + tk::dnn::Activation a86(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c87(&net, 512, 1, 1, 1, 1, 0, 0, c87_bin, true); + tk::dnn::Activation a87(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r88_layers[1] = {&a86}; + tk::dnn::Route r88(&net, r88_layers, 1); + + tk::dnn::Conv2d c89(&net, 512, 1, 1, 1, 1, 0, 0, c89_bin, true); + tk::dnn::Activation a89(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c90(&net, 512, 1, 1, 1, 1, 0, 0, c90_bin, true); + tk::dnn::Activation a90(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c91(&net, 512, 3, 3, 1, 1, 1, 1, c91_bin, true); + tk::dnn::Activation a91(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s92(&net, &a89); + + tk::dnn::Conv2d c93(&net, 512, 1, 1, 1, 1, 0, 0, c93_bin, true); + tk::dnn::Activation a93(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c94(&net, 512, 3, 3, 1, 1, 1, 1, c94_bin, true); + tk::dnn::Activation a94(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s95(&net, &s92); + + tk::dnn::Conv2d c96(&net, 512, 1, 1, 1, 1, 0, 0, c96_bin, true); + tk::dnn::Activation a96(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c97(&net, 512, 3, 3, 1, 1, 1, 1, c97_bin, true); + tk::dnn::Activation a97(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s98(&net, &s95); + + tk::dnn::Conv2d c99(&net, 512, 1, 1, 1, 1, 0, 0, c99_bin, true); + tk::dnn::Activation a99(&net, tk::dnn::ACTIVATION_MISH); + tk::dnn::Conv2d c100(&net, 512, 3, 3, 1, 1, 1, 1, c100_bin, true); + tk::dnn::Activation a100(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Shortcut s101(&net, &s98); + + tk::dnn::Conv2d c102(&net, 512, 1, 1, 1, 1, 0, 0, c102_bin, true); + tk::dnn::Activation a102(&net, tk::dnn::ACTIVATION_MISH); + + tk::dnn::Layer *r103_layers[2] = {&a102, &a87}; + tk::dnn::Route r103(&net, r103_layers, 2); + + tk::dnn::Conv2d c104(&net, 1024, 1, 1, 1, 1, 0, 0, c104_bin, true); + tk::dnn::Activation a104(&net, tk::dnn::ACTIVATION_MISH); + + + //################ + tk::dnn::Conv2d c105(&net, 512, 1, 1, 1, 1, 0, 0, c105_bin, true); + tk::dnn::Activation a105(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c106(&net, 1024, 3, 3, 1, 1, 1, 1, c106_bin, true); + tk::dnn::Activation a106(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c107(&net, 512, 1, 1, 1, 1, 0, 0, c107_bin, true); + tk::dnn::Activation a107(&net, tk::dnn::ACTIVATION_LEAKY); + + //SPP + tk::dnn::Pooling p108(&net, 5, 5, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); + tk::dnn::Layer *r109_layers[1] = {&a107}; + tk::dnn::Route r109(&net, r109_layers, 1); + + tk::dnn::Pooling p110(&net, 9, 9, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); + tk::dnn::Layer *r111_layers[1] = {&a107}; + tk::dnn::Route r111(&net, r111_layers, 1); + + tk::dnn::Pooling p112(&net, 13, 13, 1, 1, 12, 12, tk::dnn::POOLING_MAX_FIXEDSIZE); + tk::dnn::Layer *r113_layers[4] = {&p112, &p110, &p108, &a107}; + tk::dnn::Route r113(&net, r113_layers, 4); + //END SPP + + tk::dnn::Conv2d c114(&net, 512, 1, 1, 1, 1, 0, 0, c114_bin, true); + tk::dnn::Activation a114(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c115(&net, 1024, 3, 3, 1, 1, 1, 1, c115_bin, true); + tk::dnn::Activation a115(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c116(&net, 512, 1, 1, 1, 1, 0, 0, c116_bin, true); + tk::dnn::Activation a116(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c117(&net, 256, 1, 1, 1, 1, 0, 0, c117_bin, true); + tk::dnn::Activation a117(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Upsample u118(&net, 2); + tk::dnn::Layer *r119_layers[1] = {&a85}; + tk::dnn::Route r119(&net, r119_layers, 1); + tk::dnn::Conv2d c120(&net, 256, 1, 1, 1, 1, 0, 0, c120_bin, true); + tk::dnn::Activation a120(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r121_layers[2] = {&a120,&u118}; + tk::dnn::Route r121(&net, r121_layers, 2); + + tk::dnn::Conv2d c122(&net, 256, 1, 1, 1, 1, 0, 0, c122_bin, true); + tk::dnn::Activation a122(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c123(&net, 512, 3, 3, 1, 1, 1, 1, c123_bin, true); + tk::dnn::Activation a123(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c124(&net, 256, 1, 1, 1, 1, 0, 0, c124_bin, true); + tk::dnn::Activation a124(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c125(&net, 512, 3, 3, 1, 1, 1, 1, c125_bin, true); + tk::dnn::Activation a125(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c126(&net, 256, 1, 1, 1, 1, 0, 0, c126_bin, true); + tk::dnn::Activation a126(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c127(&net, 128, 1, 1, 1, 1, 0, 0, c127_bin, true); + tk::dnn::Activation a127(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Upsample u128(&net, 2); + tk::dnn::Layer *r129_layers[1] = {&a54}; + tk::dnn::Route r129(&net, r129_layers, 1); + tk::dnn::Conv2d c130(&net, 128, 1, 1, 1, 1, 0, 0, c130_bin, true); + tk::dnn::Activation a130(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r131_layers[2] = {&a130,&u128}; + tk::dnn::Route r131(&net, r131_layers, 2); + + + tk::dnn::Conv2d c132(&net, 128, 1, 1, 1, 1, 0, 0, c132_bin, true); + tk::dnn::Activation a132(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c133(&net, 256, 3, 3, 1, 1, 1, 1, c133_bin, true); + tk::dnn::Activation a133(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c134(&net, 128, 1, 1, 1, 1, 0, 0, c134_bin, true); + tk::dnn::Activation a134(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c135(&net, 256, 3, 3, 1, 1, 1, 1, c135_bin, true); + tk::dnn::Activation a135(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c136(&net, 128, 1, 1, 1, 1, 0, 0, c136_bin, true); + tk::dnn::Activation a136(&net, tk::dnn::ACTIVATION_LEAKY); + + + tk::dnn::Conv2d c137(&net, 256, 3, 3, 1, 1, 1, 1, c137_bin, true); + tk::dnn::Activation a137(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c138(&net, 45, 1, 1, 1, 1, 0, 0, c138_bin, false); + tk::dnn::Yolo yolo139(&net, classes, 3, g139_bin, 3, 1.2); + + tk::dnn::Layer *r140_layers[1] = {&a136}; + tk::dnn::Route r140(&net, r140_layers, 1); + tk::dnn::Conv2d c141(&net, 256, 3, 3, 2, 2, 1, 1, c141_bin, true); + tk::dnn::Activation a141(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r142_layers[2] = {&a141,&a126}; + tk::dnn::Route r142(&net, r142_layers, 2); + + tk::dnn::Conv2d c143(&net, 256, 1, 1, 1, 1, 0, 0, c143_bin, true); + tk::dnn::Activation a143(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c144(&net, 512, 3, 3, 1, 1, 1, 1, c144_bin, true); + tk::dnn::Activation a144(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c145(&net, 256, 1, 1, 1, 1, 0, 0, c145_bin, true); + tk::dnn::Activation a145(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c146(&net, 512, 3, 3, 1, 1, 1, 1, c146_bin, true); + tk::dnn::Activation a146(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c147(&net, 256, 1, 1, 1, 1, 0, 0, c147_bin, true); + tk::dnn::Activation a147(&net, tk::dnn::ACTIVATION_LEAKY); + + tk::dnn::Conv2d c148(&net, 512, 3, 3, 1, 1, 1, 1, c148_bin, true); + tk::dnn::Activation a148(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c149(&net, 45, 1, 1, 1, 1, 0, 0, c149_bin, false); + tk::dnn::Yolo yolo150(&net, classes, 3, g150_bin, 3, 1.1); + + tk::dnn::Layer *r151_layers[1] = {&a147}; + tk::dnn::Route r151(&net, r151_layers, 1); + tk::dnn::Conv2d c152(&net, 512, 3, 3, 2, 2, 1, 1, c152_bin, true); + tk::dnn::Activation a152(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer *r153_layers[2] = {&a152,&a116}; + tk::dnn::Route r153(&net, r153_layers, 2); + + tk::dnn::Conv2d c154(&net, 512, 1, 1, 1, 1, 0, 0, c154_bin, true); + tk::dnn::Activation a154(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c155(&net, 1024, 3, 3, 1, 1, 1, 1, c155_bin, true); + tk::dnn::Activation a155(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c156(&net, 512, 1, 1, 1, 1, 0, 0, c156_bin, true); + tk::dnn::Activation a156(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c157(&net, 1024, 3, 3, 1, 1, 1, 1, c157_bin, true); + tk::dnn::Activation a157(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c158(&net, 512, 1, 1, 1, 1, 0, 0, c158_bin, true); + tk::dnn::Activation a158(&net, tk::dnn::ACTIVATION_LEAKY); + + tk::dnn::Conv2d c159(&net, 1024, 3, 3, 1, 1, 1, 1, c159_bin, true); + tk::dnn::Activation a159(&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Conv2d c160(&net, 45, 1, 1, 1, 1, 0, 0, c160_bin, false); + tk::dnn::Yolo yolo161(&net, classes, 3, g161_bin, 3, 1.05); + + + + + + + yolo[0] = &yolo139; + yolo[1] = &yolo150; + yolo[2] = &yolo161; + + // fill classes names + for (int i = 0; i < 3; i++) + { + yolo[i]->classesNames = {"person","car","truck","bus","motor","bike","rider","traffic light","traffic sign","train"}; + } + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + + //print network model + net.print(); + + // //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo4_berkeley")); + + // the network have 3 outputs + tk::dnn::dataDim_t out_dim[3]; + for (int i = 0; i < 3; i++) + out_dim[i] = yolo[i]->output_dim; + dnnType *cudnn_out[3], *rt_out[3]; + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + + for (int i = 0; i < 3; i++) + cudnn_out[i] = yolo[i]->dstData; + + printCenteredTitle(" compute detections ", '=', 30); + TIMER_START + int ndets = 0; + tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); + for (int i = 0; i < 3; i++) + yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); + tk::dnn::Yolo::mergeDetections(dets, ndets, classes); + + for (int j = 0; j < ndets; j++) + { + tk::dnn::Yolo::box b = dets[j].bbox; + int x0 = (b.x - b.w / 2.); + int x1 = (b.x + b.w / 2.); + int y0 = (b.y - b.h / 2.); + int y1 = (b.y + b.h / 2.); + + int cl = 0; + for (int c = 0; c < classes; ++c) + { + float prob = dets[j].prob[c]; + if (prob > 0) + cl = c; + } + std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; + } + TIMER_STOP + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + for (int i = 0; i < 3; i++) + rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; + + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for (int i = 0; i < 3; i++) + { + printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); + dnnType *out, *out_h; + int odim = out_dim[i].tot(); + readBinaryFile(output_bins[i], odim, &out_h, &out); + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} -- 2.52.0 From 377310af5073e9f56289989662e9be4d406f6c18 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 19 May 2020 09:32:27 +0200 Subject: [PATCH 007/228] Add variable batch for detector update Signed-off-by: Micaela Verucchi --- demo/demo/demo.cpp | 2 +- demo/demo/map.cpp | 2 +- include/tkDNN/DetectionNN.h | 13 ++++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index afa5577..76b451d 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -111,7 +111,7 @@ int main(int argc, char *argv[]) { break; //inference - detNN->update(batch_dnn_input); + detNN->update(batch_dnn_input, n_batch); detNN->draw(batch_frame); if(show){ diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 89f2c0f..aefc834 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -144,7 +144,7 @@ int main(int argc, char *argv[]) //inference detected_bbox.clear(); - detNN->update(batch_dnn_input, write_res_on_file, ×, write_coco_json); + detNN->update(batch_dnn_input,1,write_res_on_file, ×, write_coco_json); detNN->draw(batch_frames); detected_bbox = detNN->detected; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index 3c56490..fc9a137 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -81,7 +81,7 @@ class DetectionNN { * * @param tensor_path path to the rt file og the NN. * @param n_classes number of classes for the given dataset. - * @param n_batches number of batches to use in inference + * @param n_batches maximum number of batches to use in inference * @return true if everything is correct, false otherwise. */ virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1) = 0; @@ -90,20 +90,23 @@ class DetectionNN { * This method performs the whole detection of the NN. * * @param frames frames to run detection on. + * @param cur_batches number of batches to use in inference * @param save_times if set to true, preprocess, inference and postprocess times * are saved on a csv file, otherwise not. * @param times pointer to the output stream where to write times * @param mAP set to true only if all the probabilities for a bounding * box are needed, as in some cases for the mAP calculation */ - void update(std::vector& frames, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ + void update(std::vector& frames, const int cur_batches=1, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ if(save_times && times==nullptr) FatalError("save_times set to true, but no valid ofstream given"); + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); if(VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); { TIMER_START - for(int bi=0; biinput_dim; - dim.n = nBatches; + dim.n = cur_batches; { if(VERBOSE) dim.print(); TIMER_START @@ -129,7 +132,7 @@ class DetectionNN { batchDetected.clear(); { TIMER_START - for(int bi=0; bi Date: Tue, 19 May 2020 16:33:45 +0200 Subject: [PATCH 008/228] Fix original size Signed-off-by: Micaela Verucchi --- include/tkDNN/DetectionNN.h | 5 +++-- src/CenternetDetection.cpp | 2 +- src/MobilenetDetection.cpp | 4 ++-- src/Yolo3Detection.cpp | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index fc9a137..2e64be8 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -30,7 +30,7 @@ class DetectionNN { tk::dnn::NetworkRT *netRT = nullptr; dnnType *input_d; - cv::Size originalSize; + std::vector originalSize; cv::Scalar colors[256]; @@ -103,13 +103,14 @@ class DetectionNN { if(cur_batches > nBatches) FatalError("A batch size greater than nBatches cannot be used"); + originalSize.clear(); if(VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); { TIMER_START for(int bi=0; bipluginFactory->n_yolos; i++) rt_out[i] = (dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi; - float x_ratio = float(originalSize.width) / float(netRT->input_dim.w); - float y_ratio = float(originalSize.height) / float(netRT->input_dim.h); + float x_ratio = float(originalSize[bi].width) / float(netRT->input_dim.w); + float y_ratio = float(originalSize[bi].height) / float(netRT->input_dim.h); // compute dets nDets = 0; -- 2.52.0 From 64098ad2441540a3bb5d12cd42f199c54ba6b4ba Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 22 May 2020 12:51:25 +0200 Subject: [PATCH 009/228] Add CenterNet based on DLA34 for 3D, CUDNN and TensorRT work Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 + tests/dla34_cnet3d/dla34_cnet3d.cpp | 562 ++++++++++++++++++++++++++++ 2 files changed, 565 insertions(+) create mode 100644 tests/dla34_cnet3d/dla34_cnet3d.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 028d375..1ae4289 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,9 @@ target_link_libraries(test_dla34 tkDNN) add_executable(test_dla34_cnet tests/dla34_cnet/dla34_cnet.cpp) target_link_libraries(test_dla34_cnet tkDNN) +add_executable(test_dla34_cnet3d tests/dla34_cnet3d/dla34_cnet3d.cpp) +target_link_libraries(test_dla34_cnet3d tkDNN) + add_executable(test_imuodom tests/imuodom/imuodom.cpp) target_link_libraries(test_imuodom tkDNN) ################################################################################ diff --git a/tests/dla34_cnet3d/dla34_cnet3d.cpp b/tests/dla34_cnet3d/dla34_cnet3d.cpp new file mode 100644 index 0000000..ecd5693 --- /dev/null +++ b/tests/dla34_cnet3d/dla34_cnet3d.cpp @@ -0,0 +1,562 @@ +#include +#include "tkdnn.h" + +const char *input_bin = "dla34_cnet3d/debug/input.bin"; +const char *conv1_bin = "dla34_cnet3d/layers/base-base_layer-0.bin"; +const char *conv2_bin = "dla34_cnet3d/layers/base-level0-0.bin"; +const char *conv3_bin = "dla34_cnet3d/layers/base-level1-0.bin"; +// s - stage, t - tree +const char *s1_t1_conv1_bin = "dla34_cnet3d/layers/base-level2-tree1-conv1.bin"; +const char *s1_t1_conv2_bin = "dla34_cnet3d/layers/base-level2-tree1-conv2.bin"; +const char *s1_t1_project = "dla34_cnet3d/layers/base-level2-project-0.bin"; +const char *s1_t2_conv1_bin = "dla34_cnet3d/layers/base-level2-tree2-conv1.bin"; +const char *s1_t2_conv2_bin = "dla34_cnet3d/layers/base-level2-tree2-conv2.bin"; +const char *s1_root_conv1_bin = "dla34_cnet3d/layers/base-level2-root-conv.bin"; +const char *s2_t1_t1_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-tree1-conv1.bin"; +const char *s2_t1_t1_conv2_bin = "dla34_cnet3d/layers/base-level3-tree1-tree1-conv2.bin"; +const char *s2_t1_t1_project = "dla34_cnet3d/layers/base-level3-tree1-project-0.bin"; +const char *s2_t1_t2_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-tree2-conv1.bin"; +const char *s2_t1_t2_conv2_bin = "dla34_cnet3d/layers/base-level3-tree1-tree2-conv2.bin"; +const char *s2_t1_root_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-root-conv.bin"; +const char *s2_t2_t1_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-tree1-conv1.bin"; +const char *s2_t2_t1_conv2_bin = "dla34_cnet3d/layers/base-level3-tree2-tree1-conv2.bin"; +const char *s2_t2_t2_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-tree2-conv1.bin"; +const char *s2_t2_t2_conv2_bin = "dla34_cnet3d/layers/base-level3-tree2-tree2-conv2.bin"; +const char *s2_t2_root_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-root-conv.bin"; +const char *s3_t1_t1_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-tree1-conv1.bin"; +const char *s3_t1_t1_conv2_bin = "dla34_cnet3d/layers/base-level4-tree1-tree1-conv2.bin"; +const char *s3_t1_t1_project = "dla34_cnet3d/layers/base-level4-tree1-project-0.bin"; +const char *s3_t1_t2_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-tree2-conv1.bin"; +const char *s3_t1_t2_conv2_bin = "dla34_cnet3d/layers/base-level4-tree1-tree2-conv2.bin"; +const char *s3_t1_root_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-root-conv.bin"; +const char *s3_t2_t1_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-tree1-conv1.bin"; +const char *s3_t2_t1_conv2_bin = "dla34_cnet3d/layers/base-level4-tree2-tree1-conv2.bin"; +const char *s3_t2_t2_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-tree2-conv1.bin"; +const char *s3_t2_t2_conv2_bin = "dla34_cnet3d/layers/base-level4-tree2-tree2-conv2.bin"; +const char *s3_t2_root_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-root-conv.bin"; +const char *s4_t1_conv1_bin = "dla34_cnet3d/layers/base-level5-tree1-conv1.bin"; +const char *s4_t1_conv2_bin = "dla34_cnet3d/layers/base-level5-tree1-conv2.bin"; +const char *s4_t1_project = "dla34_cnet3d/layers/base-level5-project-0.bin"; +const char *s4_t2_conv1_bin = "dla34_cnet3d/layers/base-level5-tree2-conv1.bin"; +const char *s4_t2_conv2_bin = "dla34_cnet3d/layers/base-level5-tree2-conv2.bin"; +const char *s4_root_conv1_bin = "dla34_cnet3d/layers/base-level5-root-conv.bin"; + +//final +// const char *fc_bin = "dla34_cnet3d/layers/output.bin"; + +const char *ida_0_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_0-proj_1-conv.bin"; +const char *ida_0_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin"; +const char *ida_0_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_0-up_1.bin"; +const char *ida_0_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_0-node_1-conv.bin"; +const char *ida_0_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin"; + +const char *ida_1_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_1-conv.bin"; +const char *ida_1_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin"; +const char *ida_1_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_1-up_1.bin"; +const char *ida_1_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_1-conv.bin"; +const char *ida_1_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin"; +const char *ida_1_p_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_2-conv.bin"; +const char *ida_1_p_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin"; +const char *ida_1_up_2_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_1-up_2.bin"; +const char *ida_1_n_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_2-conv.bin"; +const char *ida_1_n_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin"; + +const char *ida_2_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_1-conv.bin"; +const char *ida_2_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin"; +const char *ida_2_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_1.bin"; +const char *ida_2_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_1-conv.bin"; +const char *ida_2_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin"; +const char *ida_2_p_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_2-conv.bin"; +const char *ida_2_p_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin"; +const char *ida_2_up_2_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_2.bin"; +const char *ida_2_n_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_2-conv.bin"; +const char *ida_2_n_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin"; +const char *ida_2_p_3_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_3-conv.bin"; +const char *ida_2_p_3_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin"; +const char *ida_2_up_3_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_3.bin"; +const char *ida_2_n_3_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_3-conv.bin"; +const char *ida_2_n_3_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin"; + +const char *ida_up_p_1_dcn_bin = "dla34_cnet3d/layers/ida_up-proj_1-conv.bin"; +const char *ida_up_p_1_conv_bin = "dla34_cnet3d/layers/ida_up-proj_1-conv-conv_offset_mask.bin"; +const char *ida_up_up_1_deconv_bin = "dla34_cnet3d/layers/ida_up-up_1.bin"; +const char *ida_up_n_1_dcn_bin = "dla34_cnet3d/layers/ida_up-node_1-conv.bin"; +const char *ida_up_n_1_conv_bin = "dla34_cnet3d/layers/ida_up-node_1-conv-conv_offset_mask.bin"; +const char *ida_up_p_2_dcn_bin = "dla34_cnet3d/layers/ida_up-proj_2-conv.bin"; +const char *ida_up_p_2_conv_bin = "dla34_cnet3d/layers/ida_up-proj_2-conv-conv_offset_mask.bin"; +const char *ida_up_up_2_deconv_bin = "dla34_cnet3d/layers/ida_up-up_2.bin"; +const char *ida_up_n_2_dcn_bin = "dla34_cnet3d/layers/ida_up-node_2-conv.bin"; +const char *ida_up_n_2_conv_bin = "dla34_cnet3d/layers/ida_up-node_2-conv-conv_offset_mask.bin"; + +const char *hm_conv1_bin = "dla34_cnet3d/layers/hm-0.bin"; +const char *hm_conv2_bin = "dla34_cnet3d/layers/hm-2.bin"; +const char *wh_conv1_bin = "dla34_cnet3d/layers/wh-0.bin"; +const char *wh_conv2_bin = "dla34_cnet3d/layers/wh-2.bin"; +const char *reg_conv1_bin = "dla34_cnet3d/layers/reg-0.bin"; +const char *reg_conv2_bin = "dla34_cnet3d/layers/reg-2.bin"; +const char *dep_conv1_bin = "dla34_cnet3d/layers/dep-0.bin"; +const char *dep_conv2_bin = "dla34_cnet3d/layers/dep-2.bin"; +const char *rot_conv1_bin = "dla34_cnet3d/layers/rot-0.bin"; +const char *rot_conv2_bin = "dla34_cnet3d/layers/rot-2.bin"; +const char *dim_conv1_bin = "dla34_cnet3d/layers/dim-0.bin"; +const char *dim_conv2_bin = "dla34_cnet3d/layers/dim-2.bin"; + +const char *output_bin[]={ +"dla34_cnet3d/debug/hm.bin", +"dla34_cnet3d/debug/wh.bin", +"dla34_cnet3d/debug/reg.bin", +"dla34_cnet3d/debug/dep.bin", +"dla34_cnet3d/debug/rot.bin", +"dla34_cnet3d/debug/dim.bin"}; + +int main() +{ + + // downloadWeightsifDoNotExist(input_bin, "dla34_cnet3d", "https://cloud.hipert.unimore.it/s/KRZBbCQsKAtQwpZ/download"); + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); + tk::dnn::Network net(dim); + tk::dnn::Layer *last1, *last2, *last3, *last4; + tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2; + + tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true); + tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU); + base1 = &relu2; + + tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true); + tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU); + base2 = &relu3; + + // level 2 + // tree 1 + tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true); + tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true); + last2 = &s1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s1_t1_layers[1] = { base2 }; + tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1); + // downsample + tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + // project + tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true); + + tk::dnn::Shortcut s1_t1_s1(&net, last2); + tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s1_t1_relu; + // tree 2 + tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true); + tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true); + + tk::dnn::Shortcut s1_t2_s1(&net, last1); + tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2); + tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true); + tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base3 = &s1_root_relu; + + // level 3 + // tree 1 + // tree 1 + tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true); + tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true); + last2 = &s2_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s2_t1_t1_layers[1] = { base3 }; + tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s2_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true); + + tk::dnn::Shortcut s2_t1_t1_s1(&net, last2); + tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true); + tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t1_t2_s1(&net, last1); + tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2); + tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true); + tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_root_relu; + last3 = &s2_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true); + tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s2_t2_t1_s1(&net, last1); + tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true); + tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t2_t2_s1(&net, last1); + tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t2_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4); + tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true); + tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base4 = &s2_t2_root_relu; + + // level 4 + // tree 1 + // tree 1 + tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true); + tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true); + last2 = &s3_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s3_t1_t1_layers[1] = { base4 }; + tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s3_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true); + + tk::dnn::Shortcut s3_t1_t1_s1(&net, last2); + tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true); + tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t1_t2_s1(&net, last1); + tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t1_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2); + tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true); + tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_root_relu; + last3 = &s3_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true); + tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s3_t2_t1_s1(&net, last1); + tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true); + tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t2_t2_s1(&net, last1); + tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t2_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4); + tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true); + tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base5 = &s3_t2_root_relu; + + // level 5 + // tree 1 + tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true); + tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true); + last2 = &s4_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s4_t1_layers[1] = { base5 }; + tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1); + // downsample + tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s4_t1_maxpool1; + // project + tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true); + + tk::dnn::Shortcut s4_t1_s1(&net, last2); + tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s4_t1_relu; + + // tree 2 + tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true); + tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true); + + tk::dnn::Shortcut s4_t2_s1(&net, last1); + tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s4_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 }; + tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3); + tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true); + tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base6 = &s4_root_relu; + + //final + // tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE); + // tk::dnn::Dense fc(&net, 1000, fc_bin); + + //ida 0 + tk::dnn::DeformConv2d ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true); + tk::dnn::Activation ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256); + tk::dnn::Shortcut ida_0_shortcut(&net, base5); + tk::dnn::DeformConv2d ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true); + tk::dnn::Activation ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida1 = &ida_0_n_1_relu; + + //ida1-1 + tk::dnn::Layer *route_ida1_layers_1[1] = { base5 }; + tk::dnn::Route route_ida1_1(&net, route_ida1_layers_1, 1); + + tk::dnn::DeformConv2d ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true); + tk::dnn::Activation ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut1(&net, base4); + tk::dnn::DeformConv2d ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true); + tk::dnn::Activation ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_1 = &ida_1_n_1_relu; + + //ida1-2 + tk::dnn::Layer *route_ida1_layers_2[1] = { ida1 }; + tk::dnn::Route route_ida1_2(&net, route_ida1_layers_2, 1); + + tk::dnn::DeformConv2d ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true); + tk::dnn::Activation ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut2(&net, ida2_1); + tk::dnn::DeformConv2d ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true); + tk::dnn::Activation ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_2 = &ida_1_n_2_relu; + + //ida2-1 + tk::dnn::Layer *route_ida2_layers_1[1] = { base4 }; + tk::dnn::Route route_ida2_1(&net, route_ida2_layers_1, 1); + + tk::dnn::DeformConv2d ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true); + tk::dnn::Activation ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut1(&net, base3); + tk::dnn::DeformConv2d ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true); + tk::dnn::Activation ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_1 = &ida_2_n_1_relu; + + //ida2-2 + tk::dnn::Layer *route_ida2_layers_2[1] = { ida2_1 }; + tk::dnn::Route route_ida2_2(&net, route_ida2_layers_2, 1); + + tk::dnn::DeformConv2d ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true); + tk::dnn::Activation ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut2(&net, ida3_1); + tk::dnn::DeformConv2d ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true); + tk::dnn::Activation ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_2 = &ida_2_n_2_relu; + + //ida2-3 + tk::dnn::Layer *route_ida2_layers_3[1] = { ida2_2 }; + tk::dnn::Route route_ida2_3(&net, route_ida2_layers_3, 1); + + tk::dnn::DeformConv2d ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true); + tk::dnn::Activation ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut3(&net, ida3_2); + tk::dnn::DeformConv2d ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true); + tk::dnn::Activation ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_3 = &ida_2_n_3_relu; + + //idaup-1 + tk::dnn::Layer *route_idaup_layers_1[1] = { ida2_2 }; + tk::dnn::Route route_idaup_1(&net, route_idaup_layers_1, 1); + + tk::dnn::DeformConv2d idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true); + tk::dnn::Activation idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut1(&net, ida3_3); + tk::dnn::DeformConv2d idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true); + tk::dnn::Activation idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_1 = &idaup_n_1_relu; + + //idaup-2 + tk::dnn::Layer *route_idaup_layers_2[1] = { ida1 }; + tk::dnn::Route route_idaup_2(&net, route_idaup_layers_2, 1); + + tk::dnn::DeformConv2d idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true); + tk::dnn::Activation idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut2(&net, idaup_1); + tk::dnn::DeformConv2d idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true); + tk::dnn::Activation idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_2 = &idaup_n_2_relu; + + tk::dnn::Layer *route_1_0_layers[1] = { idaup_2 }; + + // hm + tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false); + tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false); + hm->setFinal(); + int kernel = 3; + int pad = (kernel - 1)/2; + tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX); + hmax->setFinal(); + + // wh + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false); + tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false); + wh->setFinal(); + + // reg + tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false); + tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false); + reg->setFinal(); + + // dep + tk::dnn::Route *route_3_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dep_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dep_conv1_bin, false); + tk::dnn::Activation *dep_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dep = new tk::dnn::Conv2d(&net, 1, 1, 1, 1, 1, 0, 0, dep_conv2_bin, false); + dep->setFinal(); + + // rot + tk::dnn::Route *route_4_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *rot_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, rot_conv1_bin, false); + tk::dnn::Activation *rot_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *rot = new tk::dnn::Conv2d(&net, 8, 1, 1, 1, 1, 0, 0, rot_conv2_bin, false); + rot->setFinal(); + + // dim + tk::dnn::Route *route_5_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dim_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dim_conv1_bin, false); + tk::dnn::Activation *dim_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dim_ = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, dim_conv2_bin, false); + dim_->setFinal(); + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + //printDeviceVector(64, data, true); + + //print network model + net.print(); + + //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet3d")); + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + tk::dnn::Layer *outs[6] = { hm, wh, reg, dep, rot, dim_ }; + int out_count = 1; + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for(int i=0; i<6; i++) { + printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i+out_count]; + // there is the maxpool. It isn't an output but it is necessary for the process section + if(i==0) + out_count ++; + + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} -- 2.52.0 From ba8c28238433922d6699abfd82ee19875bb180e5 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 27 May 2020 18:03:09 +0200 Subject: [PATCH 010/228] Add 3D CenterNet detection class Signed-off-by: Davide Sapienza --- include/tkDNN/CenternetDetection3D.h | 100 ++++++ include/tkDNN/DetectionNN3D.h | 136 +++++++ include/tkDNN/Layer.h | 10 + include/tkDNN/kernelsThrust.h | 2 + src/CenternetDetection3D.cpp | 517 +++++++++++++++++++++++++++ src/kernels/postprocessing.cu | 14 + 6 files changed, 779 insertions(+) create mode 100644 include/tkDNN/CenternetDetection3D.h create mode 100644 include/tkDNN/DetectionNN3D.h create mode 100644 src/CenternetDetection3D.cpp diff --git a/include/tkDNN/CenternetDetection3D.h b/include/tkDNN/CenternetDetection3D.h new file mode 100644 index 0000000..7b9ea7a --- /dev/null +++ b/include/tkDNN/CenternetDetection3D.h @@ -0,0 +1,100 @@ +#ifndef CENTERNETDETECTION3D_H +#define CENTERNETDETECTION3D_H + +#include "kernels.h" +#include +#include "opencv2/opencv.hpp" +#include +#include +#include // std::iota +#include // std::sort + +#include "DetectionNN3D.h" + +#include "kernelsThrust.h" + + +namespace tk { namespace dnn { + +class CenternetDetection3D : public DetectionNN3D +{ +private: + tk::dnn::dataDim_t dim; + tk::dnn::dataDim_t dim2; + tk::dnn::dataDim_t dim_hm; + tk::dnn::dataDim_t dim_wh; + tk::dnn::dataDim_t dim_reg; + tk::dnn::dataDim_t dim_dep; + tk::dnn::dataDim_t dim_rot; + tk::dnn::dataDim_t dim_dim; + float *topk_scores; + int *topk_inds_; + float *topk_ys_; + float *topk_xs_; + int *ids_d, *ids_; + + float *ones; + + float *scores, *scores_d; + int *clses, *clses_d; + int *topk_inds_d; + float *topk_ys_d; + float *topk_xs_d; + int *inttopk_xs_d, *inttopk_ys_d; + + float *xs, *ys; + + float *dep, *rot, *dim_, *wh; + float *dep_d, *rot_d, *dim_d, *wh_d; + + float *target_coords; + + #ifdef OPENCV_CUDACONTRIB + float *mean_d; + float *stddev_d; + #else + cv::Vec mean; + cv::Vec stddev; + dnnType *input; + #endif + cv::Mat r; + cv::Mat calibs; + float *d_ptrs; + + cv::Mat src; + cv::Mat dst; + cv::Mat dst2; + cv::Mat trans, trans2; + //processing + int K = 100; + int width = 128;//56; // TODO + + // pointer used in the kernels + float *src_out; + int *ids_out; + + struct threshold op; + float peakThreshold = 0.2; + float centerThreshold = 0.3; //default 0.5 + cv::Mat corners, pts3DHomo; + + std::vector detected3D; + std::vectorcls3D; + std::vector> face_id; + +public: + CenternetDetection3D() {}; + ~CenternetDetection3D() {}; + + bool init(const std::string& tensor_path, const int n_classes=3); + void preprocess(cv::Mat &frame); + void postprocess(); + cv::Mat draw(cv::Mat &frame); +}; + + +} // namespace dnn +} // namespace tk + + +#endif /*CENTERNETDETECTION_H*/ \ No newline at end of file diff --git a/include/tkDNN/DetectionNN3D.h b/include/tkDNN/DetectionNN3D.h new file mode 100644 index 0000000..111d7e2 --- /dev/null +++ b/include/tkDNN/DetectionNN3D.h @@ -0,0 +1,136 @@ +#ifndef DETECTIONNN3D_H +#define DETECTIONNN3D_H + +#include +#include +#include +#include +#include +#include "utils.h" + +#include +#include +#include + +#include "tkdnn.h" + +// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. + +#ifdef OPENCV_CUDACONTRIB +#include +#include +#endif + + +namespace tk { namespace dnn { + +class DetectionNN3D { + + protected: + tk::dnn::NetworkRT *netRT = nullptr; + dnnType *input_d; + + cv::Size originalSize; + + cv::Scalar colors[256]; + +#ifdef OPENCV_CUDACONTRIB + cv::cuda::GpuMat bgr[3]; + cv::cuda::GpuMat imagePreproc; +#else + cv::Mat bgr[3]; + cv::Mat imagePreproc; + dnnType *input; +#endif + + /** + * This method preprocess the image, before feeding it to the NN. + * + * @param frame original frame to adapt for inference. + */ + virtual void preprocess(cv::Mat &frame) = 0; + + /** + * This method postprocess the output of the NN to obtain the correct + * boundig boxes. + * + */ + virtual void postprocess() = 0; + + public: + int classes = 0; + float confThreshold = 0.3; /*threshold on the confidence of the boxes*/ + + std::vector detected; /*bounding boxes in output*/ + std::vector stats; /*keeps track of inference times (ms)*/ + std::vector classesNames; + + DetectionNN3D() {}; + ~DetectionNN3D(){}; + + /** + * Method used to inialize the class, allocate memory and compute + * needed data. + * + * @param tensor_path path to the rt file og the NN. + * @param n_classes number of classes for the given dataset. + * @return true if everything is correct, false otherwise. + */ + virtual bool init(const std::string& tensor_path, const int n_classes=3) = 0; + + /** + * Method to draw boundixg boxes and labels on a frame. + * + * @param frame orginal frame to draw bounding box on. + * @return frame with boundig boxes. + */ + virtual cv::Mat draw(cv::Mat &frame){}; + + /** + * This method performs the whole detection of the NN. + * + * @param frame frame to run detection on. + * @param save_times if set to true, preprocess, inference and postprocess times + * are saved on a csv file, otherwise not. + * @param times pointer to the output stream where to write times + */ + void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){ + if(!frame.data) + FatalError("No image data feed to detection"); + + if(save_times && times==nullptr) + FatalError("save_times set to true, but no valid ofstream given"); + + originalSize = frame.size(); + printCenteredTitle(" TENSORRT detection ", '=', 30); + { + TIMER_START + preprocess(frame); + TIMER_STOP + if(save_times) *times<input_dim; + { + dim.print(); + TIMER_START + netRT->infer(dim, input_d); + TIMER_STOP + dim.print(); + stats.push_back(t_ns); + if(save_times) *times< corners; + float prob; + + void print() + { + std::cout<<"\tcl: "<input_dim; + + const char *kitti_class_name[] = { + "person", "car", "bicycle"}; + classesNames = std::vector(kitti_class_name, std::end( kitti_class_name)); + + for(int c=0; cinput_dim.tot())); + + dim_hm = tk::dnn::dataDim_t(1, 3, 128, 128, 1); + dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1); + dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1); + dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1); + + checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) ); + checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); + checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); + for(int i =0; iinput_dim.tot())); + mean << 0.485, 0.456, 0.406; + stddev << 0.229, 0.224, 0.225; +#endif + + calibs = cv::Mat(cv::Size(4,3), CV_32F); + calibs.at(0,0) = 707.0493; + calibs.at(0,1) = 0.0; + calibs.at(0,2) = 604.0814; + calibs.at(0,3) = 45.75831; + calibs.at(1,0) = 0.0; + calibs.at(1,1) = 707.0493; + calibs.at(1,2) = 180.5066; + calibs.at(1,3) = -0.3454157; + calibs.at(2,0) = 0.0; + calibs.at(2,1) = 0.0; + calibs.at(2,2) = 1.0; + calibs.at(2,3) = 0.004981016; + + r = cv::Mat(cv::Size(3,3), CV_32F); + r.at(0,1) = 0.0; + r.at(1,0) = 0.0; + r.at(1,1) = 1.0; + r.at(1,2) = 0.0; + r.at(2,1) = 0.0; + + corners = cv::Mat(cv::Size(8,3), CV_32F); + corners.at(1,0) = 0.0; + corners.at(1,1) = 0.0; + corners.at(1,2) = 0.0; + corners.at(1,3) = 0.0; + + pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F); + pts3DHomo.at(3,0) = 1.0; + pts3DHomo.at(3,1) = 1.0; + pts3DHomo.at(3,2) = 1.0; + pts3DHomo.at(3,3) = 1.0; + pts3DHomo.at(3,4) = 1.0; + pts3DHomo.at(3,5) = 1.0; + pts3DHomo.at(3,6) = 1.0; + pts3DHomo.at(3,7) = 1.0; + + checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) ); + + // Alloc array used in the kernel + checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); + + dst2.at(0,0)=width * 0.5; + dst2.at(0,1)=width * 0.5; + dst2.at(1,0)=width * 0.5; + dst2.at(1,1)=width * 0.5 + width * -0.5; + + dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); + dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); + + face_id.push_back({0,1,5,4}); + face_id.push_back({1,2,6, 5}); + face_id.push_back({2,3,7,6}); + face_id.push_back({3,0,4,7}); + // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); +} + +void CenternetDetection3D::preprocess(cv::Mat &frame){ + // -----------------------------------pre-process ------------------------------------------ + + // auto start_t = std::chrono::steady_clock::now(); + // auto step_t = std::chrono::steady_clock::now(); + // auto end_t = std::chrono::steady_clock::now(); + cv::Size sz = originalSize; + // std::cout<<"image: "< 0 + + src.at(0,0)=c[0]; + src.at(0,1)=c[1]; + src.at(1,0)=c[0]; + src.at(1,1)=c[1] + s[0] * -0.5; + dst.at(0,0)=netRT->input_dim.w * 0.5; + dst.at(0,1)=netRT->input_dim.h * 0.5; + dst.at(1,0)=netRT->input_dim.w * 0.5; + dst.at(1,1)=netRT->input_dim.h * 0.5 + netRT->input_dim.w * -0.5; + + src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); + src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); + dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); + dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); + + trans = cv::getAffineTransform( src, dst ); + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME gett affine trans: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + trans2 = cv::getAffineTransform( dst2, src ); + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + } + sz_old = sz; +#ifdef OPENCV_CUDACONTRIB + std::cout<<"OPENCV CPMTROB\n"; + cv::cuda::GpuMat im_Orig; + cv::cuda::GpuMat imageF1_d, imageF2_d; + + im_Orig = cv::cuda::GpuMat(frame); + // cv::cuda::resize (im_Orig, imageF1_d, cv::Size(new_width, new_height)); + imageF1_d = im_Orig; + checkCuda( cudaDeviceSynchronize() ); + + sz = imageF1_d.size(); + // std::cout<<"size: "<(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR ); + checkCuda( cudaDeviceSynchronize() ); + + imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0); + checkCuda( cudaDeviceSynchronize() ); + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME convert: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + dim2 = dim; + cv::cuda::GpuMat bgr[3]; + cv::cuda::split(imageF1_d,bgr);//split source + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME split: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + for(int i=0; i(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + checkCuda(cudaMemcpy(input_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; +#else + std::cout<<"NO OPENCV CPMTROB\n"; + cv::Mat imageF; + // resize(frame, imageF, cv::Size(new_width, new_height)); + imageF = frame; + sz = imageF.size(); + // std::cout<<"size: "<(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + cv::Mat trans = cv::getAffineTransform( src, dst ); + cv::warpAffine(imageF, imageF, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR ); + // end_t = std::chrono::steady_clock::now(); + // std::cout << " TIME warpAffine: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + + sz = imageF.size(); + // std::cout<<"size: "<(end_t - step_t).count() << " us" << std::endl; + // step_t = end_t; + dim2 = dim; + //split channels + cv::Mat bgr[3]; + cv::split(imageF,bgr);//split source + for(int i=0; i<3; i++){ + bgr[i] = bgr[i] - mean[i]; + bgr[i] = bgr[i] / stddev[i]; + } + + //write channels + for(int i=0; ibuffersRT[1]; + rt_out[1] = (dnnType *)netRT->buffersRT[2]; + rt_out[2] = (dnnType *)netRT->buffersRT[3]; + rt_out[3] = (dnnType *)netRT->buffersRT[4]; + rt_out[4] = (dnnType *)netRT->buffersRT[5]; + rt_out[5] = (dnnType *)netRT->buffersRT[6]; + rt_out[6] = (dnnType *)netRT->buffersRT[7]; + + // ------------------------------------ process -------------------------------------------- + activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot()); + checkCuda( cudaDeviceSynchronize() ); + + // output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1. + activationSIGMOIDForward(rt_out[4], rt_out[4], dim_dep.tot()); + checkCuda( cudaDeviceSynchronize() ); + transformDep(ones, ones + dim_dep.tot(), rt_out[4], rt_out[4] + dim_dep.tot()); + checkCuda( cudaDeviceSynchronize() ); + + subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op); + + // ----------- nms end + // ----------- topk + + if(K > dim_hm.h * dim_hm.w){ + printf ("Error topk (K is too large)\n"); + return; + } + + checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) ); + + sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d); + checkCuda( cudaDeviceSynchronize() ); + + topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d); + checkCuda( cudaDeviceSynchronize() ); + + checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) ); + + topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d); + + checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); + + checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); + + // ----------- topk end + + topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out); + // checkCuda( cudaDeviceSynchronize() ); + + + getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[4], dep_d, ids_out); + checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[5], rot_d, ids_out); + checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[6], dim_d, ids_out); + checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_wh.c, dim_wh.h * dim_wh.w, rt_out[2], wh_d, ids_out); + checkCuda( cudaMemcpy(wh, wh_d, K * dim_wh.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + checkCuda( cudaMemcpy(xs, topk_xs_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(ys, topk_ys_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + + // ---------------------------------- post-process ----------------------------------------- + + // ddd_post_process_2d + cv::Mat new_pt1(cv::Size(1,2), CV_32F); + cv::Mat new_pt2(cv::Size(1,2), CV_32F); + + for(int i = 0; i(0,0)=static_cast(trans2.at(0,0))*xs[i] + + static_cast(trans2.at(0,1))*ys[i] + + static_cast(trans2.at(0,2))*1.0; + new_pt1.at(0,1)=static_cast(trans2.at(1,0))*xs[i] + + static_cast(trans2.at(1,1))*ys[i] + + static_cast(trans2.at(1,2))*1.0; + + new_pt2.at(0,0)=static_cast(trans2.at(0,0))*wh[i] + + static_cast(trans2.at(0,1))*wh[K+i] + + static_cast(trans2.at(0,2))*1.0; + new_pt2.at(0,1)=static_cast(trans2.at(1,0))*wh[i] + + static_cast(trans2.at(1,1))*wh[K+i] + + static_cast(trans2.at(1,2))*1.0; + + target_coords[i*4] = new_pt1.at(0,0); + target_coords[i*4+1] = new_pt1.at(0,1); + target_coords[i*4+2] = new_pt2.at(0,0); + target_coords[i*4+3] = new_pt2.at(0,1); + } + + float alpha; + float x, y, z, rot_y; + detected3D.clear(); + for(int i = 0; i rot[5*K + j]) + alpha = std::atan2(rot[2*K + j], rot[3*K + j]) -0.5 * M_PI; + else + alpha = std::atan2(rot[6*K + j], rot[7*K + j]) +0.5 * M_PI; + + // unproject_2d_to_3d + z = dep[j] - calibs.at(2,3);// z = depth - P[2, 3] + x = (target_coords[j*4] * dep[j] - calibs.at(0,3) - calibs.at(0,2) * z) / calibs.at(0,0); + y = (target_coords[j*4+1] * dep[j] - calibs.at(1,3) - calibs.at(1,2) * z) / calibs.at(1,1) + (dim_[j] / 2); + // alpha2rot_y + rot_y = (alpha + std::atan2(target_coords[j*4] - calibs.at(0,2), calibs.at(0,0))); + if(rot_y>M_PI) + rot_y -= 2*M_PI; + if(rot_y peakThreshold) { + if(scores[j] > centerThreshold) { + if(z>0) { + // compute_box_3d + r.at(0,0) = std::cos(rot_y); + r.at(0,2) = std::sin(rot_y); + r.at(2,0) = -std::sin(rot_y); + r.at(2,2) = std::cos(rot_y); + + corners.at(0,0) = dim_[2*K+j]/2; + corners.at(0,1) = dim_[2*K+j]/2; + corners.at(0,2) = -dim_[2*K+j]/2; + corners.at(0,3) = -dim_[2*K+j]/2; + corners.at(0,4) = dim_[2*K+j]/2; + corners.at(0,5) = dim_[2*K+j]/2; + corners.at(0,6) = -dim_[2*K+j]/2; + corners.at(0,7) = -dim_[2*K+j]/2; + + corners.at(1,4) = -dim_[j]; + corners.at(1,5) = -dim_[j]; + corners.at(1,6) = -dim_[j]; + corners.at(1,7) = -dim_[j]; + + corners.at(2,0) = dim_[K+j]/2; + corners.at(2,1) = -dim_[K+j]/2; + corners.at(2,2) = -dim_[K+j]/2; + corners.at(2,3) = dim_[K+j]/2; + corners.at(2,4) = dim_[K+j]/2; + corners.at(2,5) = -dim_[K+j]/2; + corners.at(2,6) = -dim_[K+j]/2; + corners.at(2,7) = dim_[K+j]/2; + cv::Mat aus = r * corners; + + for(int k=0; k<8; k++) { + aus.at(0,k) += x; + aus.at(1,k) += y; + aus.at(2,k) += z; + } + // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); + for(int k1=0; k1<3; k1++) { + for(int k2=0; k2<8; k2++) + pts3DHomo.at(k1,k2) = aus.at(k1,k2); + } + aus.release(); + aus = calibs * pts3DHomo; + + tk::dnn::box3D res; + for(int k=0; k<8; k++) { + res.corners.push_back(aus.at(0,k) / aus.at(2,k)); + res.corners.push_back(aus.at(1,k) / aus.at(2,k)); + } + res.cl = i; + res.prob = scores[j]; + res.print(); + detected3D.push_back(res); + } + } + } + } + } +} + +cv::Mat CenternetDetection3D::draw(cv::Mat &frame) { + tk::dnn::box3D b; + int x0, w, x1, y0, h, y1; + int objClass; + std::string det_class; + + int baseline = 0; + float font_scale = 0.5; + int thickness = 2; + + // draw dets + for(int i=0; i=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), + b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + colors[b.cl], 2); + if(ind_f == 0) { + cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), + b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), + b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); + cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), + b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), + b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + } + } + } + // draw label + cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); + cv::rectangle(frame, cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1)), + cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), + (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); + cv::putText(frame, classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); + } + return frame; +} + +}} + + diff --git a/src/kernels/postprocessing.cu b/src/kernels/postprocessing.cu index 3510200..88dbcc1 100644 --- a/src/kernels/postprocessing.cu +++ b/src/kernels/postprocessing.cu @@ -1,5 +1,11 @@ #include "kernelsThrust.h" +void transformDep(float *src_begin, float *src_end, float *dst_begin, float *dst_end) { + int e = exp(-6); + thrust::transform(thrust::device, dst_begin, dst_end, thrust::make_constant_iterator(e), dst_begin, thrust::plus()); + thrust::transform(thrust::device, src_begin, src_end, dst_begin, dst_begin, thrust::divides()); + thrust::transform(thrust::device, dst_begin, dst_end, thrust::make_constant_iterator(-1.0), dst_begin, thrust::plus()); +} void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op){ thrust::transform(thrust::device, src_begin, src_end, src2_begin, src_out, op); @@ -51,6 +57,14 @@ void topKxyAddOffset(int * ids_begin, const int K, const int size, thrust::transform(thrust::device, intys_begin, intys_begin + K, src_out, ys_begin, thrust::plus()); } +void getRecordsFromTopKId(int * ids_begin, const int K, const int ch, const int size, dnnType *src_begin, float *src_out, int *ids_out) { + for(int i=0; i()); + thrust::gather(thrust::device, ids_out, ids_out + K, src_begin, src_out+i*K); + } +} + void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out){ -- 2.52.0 From 6bdf47bae60acb200d40c33b9c6a85b2013b920f Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 27 May 2020 18:05:38 +0200 Subject: [PATCH 011/228] Add 3D demo program Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 ++ demo/demo/demo3D.cpp | 103 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 demo/demo/demo3D.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ae4289..0e75e3b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,6 +159,9 @@ target_link_libraries(map_demo tkDNN) add_executable(demo demo/demo/demo.cpp) target_link_libraries(demo tkDNN) +add_executable(demo3D demo/demo/demo3D.cpp) +target_link_libraries(demo3D tkDNN) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp new file mode 100644 index 0000000..e838395 --- /dev/null +++ b/demo/demo/demo3D.cpp @@ -0,0 +1,103 @@ +#include +#include +#include /* srand, rand */ +#include +#include + +#include "CenternetDetection3D.h" + +bool gRun; +bool SAVE_RESULT = false; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + std::cout<<"detection\n"; + signal(SIGINT, sig_handler); + + + std::string net = "dla34_cnet3d_fp32.rt"; + if(argc > 1) + net = argv[1]; + std::string input = "../demo/yolo_test.mp4"; + if(argc > 2) + input = argv[2]; + char ntype = 'c'; + if(argc > 3) + ntype = argv[3][0]; + int n_classes = 3; + if(argc > 4) + n_classes = atoi(argv[4]); + + tk::dnn::CenternetDetection3D cnet; + + tk::dnn::DetectionNN3D *detNN; + + switch(ntype) + { + case 'c': + detNN = &cnet; + break; + default: + FatalError("Network type not allowed (3rd parameter)\n"); + } + + detNN->init(net, n_classes); + + gRun = true; + + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + + cv::Mat frame; + cv::Mat dnn_input; + cv::namedWindow("detection", cv::WINDOW_NORMAL); + + std::vector detected_bbox; + + while(gRun) { + cap >> frame; + if(!frame.data) { + break; + } + + // this will be resized to the net format + dnn_input = frame.clone(); + + //inference + detNN->update(dnn_input); + frame = detNN->draw(frame); + + cv::imshow("detection", frame); + cv::waitKey(1); + if(SAVE_RESULT) + resultVideo << frame; + } + + std::cout<<"detection end\n"; + double mean = 0; + + std::cout<stats.begin(), detNN->stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n"; + for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); + std::cout<<"Avg: "< Date: Wed, 27 May 2020 18:06:26 +0200 Subject: [PATCH 012/228] Update README Signed-off-by: Davide Sapienza --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index aeb3e02..d700d02 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,19 @@ N.b. By default it is used FP32 inference ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) +### Run the 3D demo + +To run the 3D object detection demo follow these steps (example with CenterNet based on DLA34): +``` +rm dla34_cnet3d_fp32.rt # be sure to delete(or move) old tensorRT files +./test_dla34_cnet3d # run the yolo test (is slow) +./demo3D dla34_cnet3d_fp32.rt ../demo/yolo_test.mp4 c +``` +The demo3D program takes the same parameters of the demo program: +``` +./demo +``` + ### FP16 inference To run the an object detection demo with FP16 inference follow these steps (example with yolov3): -- 2.52.0 From 7a677d5c10ed913593fefa38e6d4f3fc4065008e Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 29 May 2020 14:54:36 +0200 Subject: [PATCH 013/228] Add CenterNet based on Resnet101 for 3D, CUDNN and TensorRT work Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 + tests/resnet101_cnet3d/resnet101_cnet3d.cpp | 443 ++++++++++++++++++++ 2 files changed, 446 insertions(+) create mode 100644 tests/resnet101_cnet3d/resnet101_cnet3d.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e75e3b..e5a2b26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,9 @@ target_link_libraries(test_bdd-csresnext50-panet-spp tkDNN) add_executable(test_resnet101_cnet tests/resnet101_cnet/resnet101_cnet.cpp) target_link_libraries(test_resnet101_cnet tkDNN) +add_executable(test_resnet101_cnet3d tests/resnet101_cnet3d/resnet101_cnet3d.cpp) +target_link_libraries(test_resnet101_cnet3d tkDNN) + add_executable(test_dla34 tests/dla34/dla34.cpp) target_link_libraries(test_dla34 tkDNN) diff --git a/tests/resnet101_cnet3d/resnet101_cnet3d.cpp b/tests/resnet101_cnet3d/resnet101_cnet3d.cpp new file mode 100644 index 0000000..0089a19 --- /dev/null +++ b/tests/resnet101_cnet3d/resnet101_cnet3d.cpp @@ -0,0 +1,443 @@ +#include + +#include "kernels.h" +#include "Yolo3Detection.h" +#include "tkdnn.h" +#include +#include // std::iota +#include // std::sort +// #include "utils.h" + +const char *input_bin = "resnet101_cnet3d/debug/input.bin"; +const char *conv1_bin = "resnet101_cnet3d/layers/conv1.bin"; + +//layer1 +const char *layer1_bin[]={ +"resnet101_cnet3d/layers/layer1-0-conv1.bin", +"resnet101_cnet3d/layers/layer1-0-conv2.bin", +"resnet101_cnet3d/layers/layer1-0-conv3.bin", +"resnet101_cnet3d/layers/layer1-0-downsample-0.bin", + +"resnet101_cnet3d/layers/layer1-1-conv1.bin", +"resnet101_cnet3d/layers/layer1-1-conv2.bin", +"resnet101_cnet3d/layers/layer1-1-conv3.bin", + +"resnet101_cnet3d/layers/layer1-2-conv1.bin", +"resnet101_cnet3d/layers/layer1-2-conv2.bin", +"resnet101_cnet3d/layers/layer1-2-conv3.bin"}; + + +//layer2 +const char *layer2_bin[]={ +"resnet101_cnet3d/layers/layer2-0-conv1.bin", +"resnet101_cnet3d/layers/layer2-0-conv2.bin", +"resnet101_cnet3d/layers/layer2-0-conv3.bin", +"resnet101_cnet3d/layers/layer2-0-downsample-0.bin", + +"resnet101_cnet3d/layers/layer2-1-conv1.bin", +"resnet101_cnet3d/layers/layer2-1-conv2.bin", +"resnet101_cnet3d/layers/layer2-1-conv3.bin", + +"resnet101_cnet3d/layers/layer2-2-conv1.bin", +"resnet101_cnet3d/layers/layer2-2-conv2.bin", +"resnet101_cnet3d/layers/layer2-2-conv3.bin", + +"resnet101_cnet3d/layers/layer2-3-conv1.bin", +"resnet101_cnet3d/layers/layer2-3-conv2.bin", +"resnet101_cnet3d/layers/layer2-3-conv3.bin" +}; +//layer3 +const char *layer3_bin[]={ +"resnet101_cnet3d/layers/layer3-0-conv1.bin", +"resnet101_cnet3d/layers/layer3-0-conv2.bin", +"resnet101_cnet3d/layers/layer3-0-conv3.bin", +"resnet101_cnet3d/layers/layer3-0-downsample-0.bin", + +"resnet101_cnet3d/layers/layer3-1-conv1.bin", +"resnet101_cnet3d/layers/layer3-1-conv2.bin", +"resnet101_cnet3d/layers/layer3-1-conv3.bin", + +"resnet101_cnet3d/layers/layer3-2-conv1.bin", +"resnet101_cnet3d/layers/layer3-2-conv2.bin", +"resnet101_cnet3d/layers/layer3-2-conv3.bin", + +"resnet101_cnet3d/layers/layer3-3-conv1.bin", +"resnet101_cnet3d/layers/layer3-3-conv2.bin", +"resnet101_cnet3d/layers/layer3-3-conv3.bin", + +"resnet101_cnet3d/layers/layer3-4-conv1.bin", +"resnet101_cnet3d/layers/layer3-4-conv2.bin", +"resnet101_cnet3d/layers/layer3-4-conv3.bin", + +"resnet101_cnet3d/layers/layer3-5-conv1.bin", +"resnet101_cnet3d/layers/layer3-5-conv2.bin", +"resnet101_cnet3d/layers/layer3-5-conv3.bin", + +"resnet101_cnet3d/layers/layer3-6-conv1.bin", +"resnet101_cnet3d/layers/layer3-6-conv2.bin", +"resnet101_cnet3d/layers/layer3-6-conv3.bin", + +"resnet101_cnet3d/layers/layer3-7-conv1.bin", +"resnet101_cnet3d/layers/layer3-7-conv2.bin", +"resnet101_cnet3d/layers/layer3-7-conv3.bin", + +"resnet101_cnet3d/layers/layer3-8-conv1.bin", +"resnet101_cnet3d/layers/layer3-8-conv2.bin", +"resnet101_cnet3d/layers/layer3-8-conv3.bin", + +"resnet101_cnet3d/layers/layer3-9-conv1.bin", +"resnet101_cnet3d/layers/layer3-9-conv2.bin", +"resnet101_cnet3d/layers/layer3-9-conv3.bin", + +"resnet101_cnet3d/layers/layer3-10-conv1.bin", +"resnet101_cnet3d/layers/layer3-10-conv2.bin", +"resnet101_cnet3d/layers/layer3-10-conv3.bin", + +"resnet101_cnet3d/layers/layer3-11-conv1.bin", +"resnet101_cnet3d/layers/layer3-11-conv2.bin", +"resnet101_cnet3d/layers/layer3-11-conv3.bin", + +"resnet101_cnet3d/layers/layer3-12-conv1.bin", +"resnet101_cnet3d/layers/layer3-12-conv2.bin", +"resnet101_cnet3d/layers/layer3-12-conv3.bin", + +"resnet101_cnet3d/layers/layer3-13-conv1.bin", +"resnet101_cnet3d/layers/layer3-13-conv2.bin", +"resnet101_cnet3d/layers/layer3-13-conv3.bin", + +"resnet101_cnet3d/layers/layer3-14-conv1.bin", +"resnet101_cnet3d/layers/layer3-14-conv2.bin", +"resnet101_cnet3d/layers/layer3-14-conv3.bin", + +"resnet101_cnet3d/layers/layer3-15-conv1.bin", +"resnet101_cnet3d/layers/layer3-15-conv2.bin", +"resnet101_cnet3d/layers/layer3-15-conv3.bin", + +"resnet101_cnet3d/layers/layer3-16-conv1.bin", +"resnet101_cnet3d/layers/layer3-16-conv2.bin", +"resnet101_cnet3d/layers/layer3-16-conv3.bin", + +"resnet101_cnet3d/layers/layer3-17-conv1.bin", +"resnet101_cnet3d/layers/layer3-17-conv2.bin", +"resnet101_cnet3d/layers/layer3-17-conv3.bin", + +"resnet101_cnet3d/layers/layer3-18-conv1.bin", +"resnet101_cnet3d/layers/layer3-18-conv2.bin", +"resnet101_cnet3d/layers/layer3-18-conv3.bin", + +"resnet101_cnet3d/layers/layer3-19-conv1.bin", +"resnet101_cnet3d/layers/layer3-19-conv2.bin", +"resnet101_cnet3d/layers/layer3-19-conv3.bin", + +"resnet101_cnet3d/layers/layer3-20-conv1.bin", +"resnet101_cnet3d/layers/layer3-20-conv2.bin", +"resnet101_cnet3d/layers/layer3-20-conv3.bin", + +"resnet101_cnet3d/layers/layer3-21-conv1.bin", +"resnet101_cnet3d/layers/layer3-21-conv2.bin", +"resnet101_cnet3d/layers/layer3-21-conv3.bin", + +"resnet101_cnet3d/layers/layer3-22-conv1.bin", +"resnet101_cnet3d/layers/layer3-22-conv2.bin", +"resnet101_cnet3d/layers/layer3-22-conv3.bin"}; + + +//layer4 +const char *layer4_bin[]={ +"resnet101_cnet3d/layers/layer4-0-conv1.bin", +"resnet101_cnet3d/layers/layer4-0-conv2.bin", +"resnet101_cnet3d/layers/layer4-0-conv3.bin", +"resnet101_cnet3d/layers/layer4-0-downsample-0.bin", + +"resnet101_cnet3d/layers/layer4-1-conv1.bin", +"resnet101_cnet3d/layers/layer4-1-conv2.bin", +"resnet101_cnet3d/layers/layer4-1-conv3.bin", + +"resnet101_cnet3d/layers/layer4-2-conv1.bin", +"resnet101_cnet3d/layers/layer4-2-conv2.bin", +"resnet101_cnet3d/layers/layer4-2-conv3.bin"}; + +const char *d_conv1_bin = "resnet101_cnet3d/layers/deconv_layers-0-conv_offset_mask.bin"; +const char *deform1_bin = "resnet101_cnet3d/layers/deconv_layers-0.bin"; +const char *deconv1_bin = "resnet101_cnet3d/layers/deconv_layers-3.bin"; + +const char *d_conv2_bin = "resnet101_cnet3d/layers/deconv_layers-6-conv_offset_mask.bin"; +const char *deform2_bin = "resnet101_cnet3d/layers/deconv_layers-6.bin"; +const char *deconv2_bin = "resnet101_cnet3d/layers/deconv_layers-9.bin"; + +const char *d_conv3_bin = "resnet101_cnet3d/layers/deconv_layers-12-conv_offset_mask.bin"; +const char *deform3_bin = "resnet101_cnet3d/layers/deconv_layers-12.bin"; +const char *deconv3_bin = "resnet101_cnet3d/layers/deconv_layers-15.bin"; + +const char *hm_conv1_bin = "resnet101_cnet3d/layers/hm-0.bin"; +const char *hm_conv2_bin = "resnet101_cnet3d/layers/hm-2.bin"; +const char *wh_conv1_bin = "resnet101_cnet3d/layers/wh-0.bin"; +const char *wh_conv2_bin = "resnet101_cnet3d/layers/wh-2.bin"; +const char *reg_conv1_bin = "resnet101_cnet3d/layers/reg-0.bin"; +const char *reg_conv2_bin = "resnet101_cnet3d/layers/reg-2.bin"; +const char *dep_conv1_bin = "resnet101_cnet3d/layers/dep-0.bin"; +const char *dep_conv2_bin = "resnet101_cnet3d/layers/dep-2.bin"; +const char *rot_conv1_bin = "resnet101_cnet3d/layers/rot-0.bin"; +const char *rot_conv2_bin = "resnet101_cnet3d/layers/rot-2.bin"; +const char *dim_conv1_bin = "resnet101_cnet3d/layers/dim-0.bin"; +const char *dim_conv2_bin = "resnet101_cnet3d/layers/dim-2.bin"; +//final +const char *fc_bin = "resnet101_cnet3d/layers/fc.bin"; + +const char *output_bin[]={ +"resnet101_cnet3d/debug/hm.bin", +"resnet101_cnet3d/debug/wh.bin", +"resnet101_cnet3d/debug/reg.bin", +"resnet101_cnet3d/debug/dep.bin", +"resnet101_cnet3d/debug/rot.bin", +"resnet101_cnet3d/debug/dim.bin"}; + +int main() +{ + // downloadWeightsifDoNotExist(input_bin, "resnet101_cnet3d", "https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download"); + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); + tk::dnn::Network net(dim); + + tk::dnn::Conv2d conv1(&net, 64, 7, 7, 2, 2, 3, 3, conv1_bin, true); + tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Pooling maxpool4(&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); + + + //layer 1 + int id_layer1_bin = 0; + tk::dnn::Layer *last = &maxpool4; + for(int i=0; i<3;i++) + { + tk::dnn::Conv2d *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 64, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true); + tk::dnn::Activation *relu1_0_1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv2 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, layer1_bin[id_layer1_bin++], true); + tk::dnn::Activation *relu1_0_2 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true); + if(i==0) { + tk::dnn::Layer *route_1_0_layers[1] = { last }; + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true); + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, layer1_0_conv3); + } else { + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, last); + } + tk::dnn::Activation *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + last = layer1_0_relu; + } + + // layer 2 + int id_layer2_bin = 0; + for(int i=0; i<4;i++) + { + tk::dnn::Conv2d *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 128, 1, 1, 1, 1, 0, 0, layer2_bin[id_layer2_bin++], true); + tk::dnn::Activation *relu1_0_1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv2; + if(i==0) + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 128, 3, 3, 2, 2, 1, 1, layer2_bin[id_layer2_bin++], true); + else + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 128, 3, 3, 1, 1, 1, 1, layer2_bin[id_layer2_bin++], true); + + tk::dnn::Activation *relu1_0_2 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 512, 1, 1, 1, 1, 0, 0, layer2_bin[id_layer2_bin++], true); + if(i==0) + { + tk::dnn::Layer *route_1_0_layers[1] = { last }; + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 512, 1, 1, 2, 2, 0, 0, layer2_bin[id_layer2_bin++], true); + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, layer1_0_conv3); + } + else + { + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, last); + } + tk::dnn::Activation *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + last = layer1_0_relu; + } + + // layer 3 + int id_layer3_bin = 0; + for(int i=0; i<23;i++) + { + tk::dnn::Conv2d *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer3_bin[id_layer3_bin++], true); + tk::dnn::Activation *relu1_0_1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv2; + if(i==0) + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 256, 3, 3, 2, 2, 1, 1, layer3_bin[id_layer3_bin++], true); + else + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, layer3_bin[id_layer3_bin++], true); + + tk::dnn::Activation *relu1_0_2 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 1024, 1, 1, 1, 1, 0, 0, layer3_bin[id_layer3_bin++], true); + if(i==0) + { + tk::dnn::Layer *route_1_0_layers[1] = { last }; + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 1024, 1, 1, 2, 2, 0, 0, layer3_bin[id_layer3_bin++], true); + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, layer1_0_conv3); + } + else + { + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, last); + } + tk::dnn::Activation *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + last = layer1_0_relu; + } + + // layer 4 + int id_layer4_bin = 0; + for(int i=0; i<3;i++) + { + tk::dnn::Conv2d *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 512, 1, 1, 1, 1, 0, 0, layer4_bin[id_layer4_bin++], true); + tk::dnn::Activation *relu1_0_1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv2; + if(i==0) + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 512, 3, 3, 2, 2, 1, 1, layer4_bin[id_layer4_bin++], true); + else + layer1_0_conv2 = new tk::dnn::Conv2d(&net, 512, 3, 3, 1, 1, 1, 1, layer4_bin[id_layer4_bin++], true); + + tk::dnn::Activation *relu1_0_2 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 2048, 1, 1, 1, 1, 0, 0, layer4_bin[id_layer4_bin++], true); + if(i==0) + { + tk::dnn::Layer *route_1_0_layers[1] = { last }; + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 2048, 1, 1, 2, 2, 0, 0, layer4_bin[id_layer4_bin++], true); + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, layer1_0_conv3); + } + else + { + tk::dnn::Shortcut *s1_0 = new tk::dnn::Shortcut(&net, last); + } + tk::dnn::Activation *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + last = layer1_0_relu; + } + + tk::dnn::DeformConv2d *layer0_deform1 = new tk::dnn::DeformConv2d(&net, 256, 1, 3, 3, 1, 1, 1, 1, deform1_bin, d_conv1_bin, true); + tk::dnn::Activation *layer0_deform1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d *layer0_deconv1 = new tk::dnn::DeConv2d(&net, 256, 4, 4, 2, 2, 1, 1, deconv1_bin, true); + tk::dnn::Activation *layer0_deconv1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::DeformConv2d *layer1_deform1 = new tk::dnn::DeformConv2d(&net, 128, 1, 3, 3, 1, 1, 1, 1, deform2_bin, d_conv2_bin, true); + tk::dnn::Activation *layer1_deform1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d *layer1_deconv1 = new tk::dnn::DeConv2d(&net, 128, 4, 4, 2, 2, 1, 1, deconv2_bin, true); + tk::dnn::Activation *layer1_deconv1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::DeformConv2d *layer2_deform1 = new tk::dnn::DeformConv2d(&net, 64, 1, 3, 3, 1, 1, 1, 1, deform3_bin, d_conv3_bin, true); + tk::dnn::Activation *layer2_deform1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d *layer2_deconv1 = new tk::dnn::DeConv2d(&net, 64, 4, 4, 2, 2, 1, 1, deconv3_bin, true); + tk::dnn::Activation *layer2_deconv1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Layer *route_1_0_layers[1] = { layer2_deconv1_relu }; + tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false); + tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false); + hm->setFinal(); + int kernel = 3; + int pad = (kernel - 1)/2; + tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX); + hmax->setFinal(); + + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false); + tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false); + wh->setFinal(); + + tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false); + tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false); + reg->setFinal(); + + // dep + tk::dnn::Route *route_3_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dep_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, dep_conv1_bin, false); + tk::dnn::Activation *dep_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dep = new tk::dnn::Conv2d(&net, 1, 1, 1, 1, 1, 0, 0, dep_conv2_bin, false); + dep->setFinal(); + + // rot + tk::dnn::Route *route_4_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *rot_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, rot_conv1_bin, false); + tk::dnn::Activation *rot_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *rot = new tk::dnn::Conv2d(&net, 8, 1, 1, 1, 1, 0, 0, rot_conv2_bin, false); + rot->setFinal(); + + // dim + tk::dnn::Route *route_5_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *dim_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, dim_conv1_bin, false); + tk::dnn::Activation *dim_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *dim_ = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, dim_conv2_bin, false); + dim_->setFinal(); + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + // printDeviceVector(64, data, true); + + //print network model + net.print(); + + //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("resnet101_cnet3d")); + + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + + // printDeviceVector(64, cudnn_out, true); + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + tk::dnn::Layer *outs[6] = { hm, wh, reg, dep, rot, dim_ }; + int out_count = 1; + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for(int i=0; i<6; i++) { + printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + // std::cout<<"OUTPUT BIN:\n"; + // printDeviceVector(odim, cudnn_out, true); + // std::cout<<"FILE BIN:\n"; + // printDeviceVector(odim, out, true); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i+out_count]; + // there is the maxpool. It isn't an output but it is necessary for the process section + if(i==0) + out_count ++; + + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} -- 2.52.0 From 3d2405323b0baa54e4796b47dc0c5e9d39e46dfc Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 29 May 2020 15:54:22 +0200 Subject: [PATCH 014/228] Add the downloading CenterNet weights and outputs for 3D Signed-off-by: Davide Sapienza --- tests/dla34_cnet3d/dla34_cnet3d.cpp | 2 +- tests/resnet101_cnet3d/resnet101_cnet3d.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dla34_cnet3d/dla34_cnet3d.cpp b/tests/dla34_cnet3d/dla34_cnet3d.cpp index ecd5693..9766b45 100644 --- a/tests/dla34_cnet3d/dla34_cnet3d.cpp +++ b/tests/dla34_cnet3d/dla34_cnet3d.cpp @@ -112,7 +112,7 @@ const char *output_bin[]={ int main() { - // downloadWeightsifDoNotExist(input_bin, "dla34_cnet3d", "https://cloud.hipert.unimore.it/s/KRZBbCQsKAtQwpZ/download"); + downloadWeightsifDoNotExist(input_bin, "dla34_cnet3d", "https://cloud.hipert.unimore.it/s/2MDyWGzQsTKMjmR/download"); // Network layout tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); diff --git a/tests/resnet101_cnet3d/resnet101_cnet3d.cpp b/tests/resnet101_cnet3d/resnet101_cnet3d.cpp index 0089a19..5d084be 100644 --- a/tests/resnet101_cnet3d/resnet101_cnet3d.cpp +++ b/tests/resnet101_cnet3d/resnet101_cnet3d.cpp @@ -194,7 +194,7 @@ const char *output_bin[]={ int main() { - // downloadWeightsifDoNotExist(input_bin, "resnet101_cnet3d", "https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download"); + downloadWeightsifDoNotExist(input_bin, "resnet101_cnet3d", "https://cloud.hipert.unimore.it/s/xH5oH9t5wdnktYf/download"); // Network layout tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); -- 2.52.0 From d936e5f740d2ead3286a907e5f3310de85e819ae Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Sat, 30 May 2020 16:57:53 +0200 Subject: [PATCH 015/228] Add mish_yashas Signed-off-by: Micaela Verucchi --- ...{activation.mish.cu => activation_mish.cu} | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) rename src/kernels/{activation.mish.cu => activation_mish.cu} (58%) diff --git a/src/kernels/activation.mish.cu b/src/kernels/activation_mish.cu similarity index 58% rename from src/kernels/activation.mish.cu rename to src/kernels/activation_mish.cu index fd55c8a..8900061 100644 --- a/src/kernels/activation.mish.cu +++ b/src/kernels/activation_mish.cu @@ -3,20 +3,39 @@ #define MISH_THRESHOLD 20 -__device__ float tanh_activate_kernel(float x){return (2/(1 + expf(-2*x)) - 1);} -__device__ float softplus_kernel(float x, float threshold = 20) { +__device__ +float tanh_activate_kernel(float x){return (2/(1 + expf(-2*x)) - 1);} + +__device__ +float softplus_kernel(float x, float threshold = 20) { if (x > threshold) return x; // too large else if (x < -threshold) return expf(x); // too small return logf(expf(x) + 1); } + + +__device__ +float mish_yashas(float x) { + float e = __expf(x); + if (x <= -18.0f) + return x * e; + + float n = e * e + 2 * e; + if (x <= -5.0f) + return x * __fdividef(n, n + 2); + + return x - 2 * __fdividef(x, n + 2); +} + // https://github.com/digantamisra98/Mish // https://github.com/AlexeyAB/darknet/blob/master/src/activation_kernels.cu __global__ void activation_mish(dnnType *input, dnnType *output, int size) { int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; if (i < size) - output[i] = input[i] * tanh_activate_kernel( softplus_kernel(input[i], MISH_THRESHOLD)); + // output[i] = input[i] * tanh_activate_kernel( softplus_kernel(input[i], MISH_THRESHOLD)); + output[i] = mish_yashas(input[i]); } /** -- 2.52.0 From 4e1c7a70b10ab2c4eb39bf1e361aa6e1bfd20ee9 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sat, 30 May 2020 16:58:40 +0200 Subject: [PATCH 016/228] darknet parser interface --- include/tkDNN/DarknetParser.h | 17 +++++++++++++++++ tests/yolo3/yolo3.cpp | 16 ++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 include/tkDNN/DarknetParser.h diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h new file mode 100644 index 0000000..bc1a8da --- /dev/null +++ b/include/tkDNN/DarknetParser.h @@ -0,0 +1,17 @@ +#pragma once +#include +#include "tkdnn.h" + +namespace tk { namespace dnn { + + tk::dnn::Network* DarknetParser(std::string cfg) { + + tk::dnn::dataDim_t dim; + tk::dnn::Network *net = new tk::dnn::Network(dim); + + + } + + + +}} diff --git a/tests/yolo3/yolo3.cpp b/tests/yolo3/yolo3.cpp index e783f31..324a4c6 100644 --- a/tests/yolo3/yolo3.cpp +++ b/tests/yolo3/yolo3.cpp @@ -1,17 +1,23 @@ #include #include #include "tkdnn.h" +#include "DarknetParser.h" int main() { - // Network layout - tk::dnn::dataDim_t dim(1, 3, 416, 416, 1); - tk::dnn::Network net(dim); + tk::dnn::Network *net = tk::dnn::DarknetParser("../../tests/yolo3/yolo3.cfg"); + // Network layout + //tk::dnn::dataDim_t dim(1, 3, 416, 416, 1); + //tk::dnn::Network net(dim); + + /* // create yolo3 model std::string bin_path = "yolo3"; downloadWeightsifDoNotExist("yolo3/layers/input.bin", bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); int classes = 80; + + tk::dnn::Yolo *yolo [3]; #include "models/Yolo3.h" @@ -30,9 +36,10 @@ int main() { //print network model net.print(); + //convert network to tensorRT tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3")); - + // the network have 3 outputs tk::dnn::dataDim_t out_dim[3]; for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; @@ -96,4 +103,5 @@ int main() { ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; } return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; + */ } -- 2.52.0 From 15105e90d30e0dc6877a9bee7686f90a31731b38 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Sat, 30 May 2020 17:25:48 +0200 Subject: [PATCH 017/228] Add parseType Signed-off-by: Micaela Verucchi --- include/tkDNN/DarknetParser.h | 13 ++++++++++--- tests/yolo3/yolo3.cpp | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index bc1a8da..c07d813 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -9,9 +9,16 @@ namespace tk { namespace dnn { tk::dnn::dataDim_t dim; tk::dnn::Network *net = new tk::dnn::Network(dim); - + } + + std::string parseType(const std::string& line){ + size_t start = line.find("["); + size_t end = line.find("]"); + if( start == std::string::npos || end == std::string::npos) + return ""; + start++; + std::string type = line.substr(start, end-start); + return type; } - - }} diff --git a/tests/yolo3/yolo3.cpp b/tests/yolo3/yolo3.cpp index 324a4c6..9187a54 100644 --- a/tests/yolo3/yolo3.cpp +++ b/tests/yolo3/yolo3.cpp @@ -6,6 +6,7 @@ int main() { tk::dnn::Network *net = tk::dnn::DarknetParser("../../tests/yolo3/yolo3.cfg"); + tk::dnn::parseType("[net]"); // Network layout //tk::dnn::dataDim_t dim(1, 3, 416, 416, 1); -- 2.52.0 From 548a3dd33c7aa8a064544e954db503109d4ee5eb Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sat, 30 May 2020 17:27:22 +0200 Subject: [PATCH 018/228] parse line by line --- include/tkDNN/DarknetParser.h | 22 +++++++++++++++++++++- tests/yolo3/yolo3.cpp | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index bc1a8da..7238c99 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -4,12 +4,32 @@ namespace tk { namespace dnn { - tk::dnn::Network* DarknetParser(std::string cfg) { + tk::dnn::Network* DarknetParser(std::string cfg_file) { tk::dnn::dataDim_t dim; tk::dnn::Network *net = new tk::dnn::Network(dim); + std::ifstream if_cfg(cfg_file); + if(!if_cfg.is_open()) + FatalError("cloud not open cfg file: " + cfg_file); + std::string line; + while(std::getline(if_cfg, line)) { + // remove comments + std::size_t found = line.find("#"); + if ( found != std::string::npos ) { + line = line.substr(0, found); + } + + // skip empty lines + if(line.size() == 0) + continue; + + std::string type = parseType(line); + if(type.size() > 0) { + std::cout<<"type: "< Date: Sat, 30 May 2020 17:57:48 +0200 Subject: [PATCH 019/228] Add darknetParseFields Signed-off-by: Micaela Verucchi --- include/tkDNN/DarknetParser.h | 41 ++++++++++++++++++++++++++++++++++- tests/yolo3/yolo3.cpp | 5 +++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index c921d2c..09d8dce 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -7,9 +7,21 @@ namespace tk { namespace dnn { struct darknetFields_t{ int width = 0; int height = 0; + int channels = 0; + int batch_normalize=0; + int filters=0; + int size=0; + int stride=0; + int pad=0; + std::string activation = ""; }; + std::ostream& operator<<(std::ostream& os, const darknetFields_t& f){ + os << f.width << " " << f.height << " " << f.channels << " " << f.batch_normalize<< " " << f.filters<< " " << f.size<< " " << f.stride << " " << f.pad << " " << f.activation; + return os; + } + std::string darknetParseType(const std::string& line){ size_t start = line.find("["); size_t end = line.find("]"); @@ -20,8 +32,35 @@ namespace tk { namespace dnn { return type; } - darknetFields_t parseFields(const std::string& line){ + bool divideNameAndValue(const std::string& line, std::string&name, std::string& value){ + size_t sep = line.find("="); + if(sep == std::string::npos) + return false; + name = line.substr(0, sep); + value = line.substr(sep+1, line.size() - (sep+1)); + return true; + } + + bool darknetParseFields(const std::string& line, darknetFields_t& fields){ + + std::string name,value; + if(!divideNameAndValue(line, name, value)) + return false; + std::cout< Date: Sat, 30 May 2020 17:59:00 +0200 Subject: [PATCH 020/228] parse layer and network --- include/tkDNN/DarknetParser.h | 58 ++++++++++++++++++++++++++++++----- tests/yolo3/yolo3.cpp | 2 +- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index c921d2c..76be28c 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -5,9 +5,10 @@ namespace tk { namespace dnn { struct darknetFields_t{ + std::string type = ""; int width = 0; int height = 0; - + int channels = 3; }; std::string darknetParseType(const std::string& line){ @@ -20,21 +21,37 @@ namespace tk { namespace dnn { return type; } - darknetFields_t parseFields(const std::string& line){ - + bool darknetParseFields(const std::string& line, darknetFields_t &fields){ + return true; } + tk::dnn::Network *darknetAddNet(darknetFields_t &fields) { + std::cout<<"Add Net: "< 0) { - std::cout<<"type: "< Date: Sat, 30 May 2020 18:00:22 +0200 Subject: [PATCH 021/228] Add some fields Signed-off-by: Micaela Verucchi --- include/tkDNN/DarknetParser.h | 8 ++++++++ tests/yolo3/yolo3.cpp | 2 ++ 2 files changed, 10 insertions(+) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index 09d8dce..d8e8b59 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -59,6 +59,14 @@ namespace tk { namespace dnn { fields.batch_normalize = std::stoi(value); else if (name == "filters") fields.filters = std::stoi(value); + else if (name == "size") + fields.size = std::stoi(value); + else if (name == "stride") + fields.stride = std::stoi(value); + else if (name == "pad") + fields.pad = std::stoi(value); + else if (name == "activation") + fields.activation = value; return true; } diff --git a/tests/yolo3/yolo3.cpp b/tests/yolo3/yolo3.cpp index f74faa5..f1602da 100644 --- a/tests/yolo3/yolo3.cpp +++ b/tests/yolo3/yolo3.cpp @@ -10,6 +10,8 @@ int main() { tk::dnn::darknetFields_t f; tk::dnn::darknetParseFields("width=40", f); tk::dnn::darknetParseFields("height=40", f); + tk::dnn::darknetParseFields("channels=40", f); + tk::dnn::darknetParseFields("activation=leaky", f); std::cout< Date: Sat, 30 May 2020 18:02:20 +0200 Subject: [PATCH 022/228] layer parser --- include/tkDNN/DarknetParser.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index 76be28c..5ba2e54 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -38,6 +38,14 @@ namespace tk { namespace dnn { std::cout<<"Add layer: "< Date: Sat, 30 May 2020 18:10:14 +0200 Subject: [PATCH 023/228] Modify darknetFields_t Signed-off-by: Micaela Verucchi --- include/tkDNN/DarknetParser.h | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index d8e8b59..8c8309f 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -9,16 +9,25 @@ namespace tk { namespace dnn { int height = 0; int channels = 0; int batch_normalize=0; + int groups = 0; int filters=0; - int size=0; - int stride=0; - int pad=0; + int size_x=0; + int size_y=0; + int stride_x=0; + int stride_y=0; + int padding_x = 0; + int padding_y = 0; + int n_mask = 0; + int classes = 0; + int num = 0; + float scale_xy = 0; + std::vector layers; std::string activation = ""; }; std::ostream& operator<<(std::ostream& os, const darknetFields_t& f){ - os << f.width << " " << f.height << " " << f.channels << " " << f.batch_normalize<< " " << f.filters<< " " << f.size<< " " << f.stride << " " << f.pad << " " << f.activation; + os << f.width << " " << f.height << " " << f.channels << " " << f.batch_normalize<< " " << f.filters << " " << " " << f.activation; return os; } @@ -59,12 +68,6 @@ namespace tk { namespace dnn { fields.batch_normalize = std::stoi(value); else if (name == "filters") fields.filters = std::stoi(value); - else if (name == "size") - fields.size = std::stoi(value); - else if (name == "stride") - fields.stride = std::stoi(value); - else if (name == "pad") - fields.pad = std::stoi(value); else if (name == "activation") fields.activation = value; -- 2.52.0 From e5e6654b1d29be9ce20c94fea5bafbc92750fa6c Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Sat, 30 May 2020 18:44:17 +0200 Subject: [PATCH 024/228] Add fields to darknetParseFields Signed-off-by: Micaela Verucchi --- include/tkDNN/DarknetParser.h | 63 +++++++++++++++++++++++++++++------ tests/yolo3/yolo3.cpp | 6 ++-- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index 8c8309f..c640d99 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -20,6 +20,7 @@ namespace tk { namespace dnn { int n_mask = 0; int classes = 0; int num = 0; + int pad = 0; float scale_xy = 0; std::vector layers; std::string activation = ""; @@ -27,7 +28,7 @@ namespace tk { namespace dnn { }; std::ostream& operator<<(std::ostream& os, const darknetFields_t& f){ - os << f.width << " " << f.height << " " << f.channels << " " << f.batch_normalize<< " " << f.filters << " " << " " << f.activation; + os << f.width << " " << f.height << " " << f.channels << " " << f.batch_normalize<< " " << f.filters << " " << f.activation<< " " << f.scale_xy; return os; } @@ -51,26 +52,68 @@ namespace tk { namespace dnn { return true; } + std::vector fromStringToIntVec(const std::string& line, const char delimiter){ + std::stringstream linestream(line); + std::string value; + std::vector values; + + while(getline(linestream,value,delimiter)) + values.push_back(std::stoi(value)); + return values; + } + bool darknetParseFields(const std::string& line, darknetFields_t& fields){ std::string name,value; if(!divideNameAndValue(line, name, value)) return false; - std::cout< Date: Sat, 30 May 2020 19:03:29 +0200 Subject: [PATCH 025/228] Add group field Signed-off-by: Micaela Verucchi --- include/tkDNN/DarknetParser.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index c640d99..be87768 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -101,6 +101,8 @@ namespace tk { namespace dnn { fields.classes = std::stoi(value); else if(name.find("num") != std::string::npos) fields.num = std::stoi(value); + else if(name.find("groups") != std::string::npos) + fields.groups = std::stoi(value); else if(name.find("scale_xy") != std::string::npos) fields.scale_xy = std::stof(value); else if(name.find("from") != std::string::npos) -- 2.52.0 From 6d81473b2a6dab2652f8f72baeb07c8c7bcf285c Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sat, 30 May 2020 22:11:00 +0200 Subject: [PATCH 026/228] yolo3 parsed ok --- include/tkDNN/DarknetParser.h | 8 +++++--- tests/yolo3/yolo3.cpp | 12 +++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index 9a1a500..61040f9 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -140,15 +140,17 @@ namespace tk { namespace dnn { if(f.type == "convolutional") { std::string wgs = wgs_path + "/c" + std::to_string(netLayers.size()) + ".bin"; printf("%d (%d,%d) (%d,%d) (%d,%d) %s %d %d\n", f.filters, f.size_x, f.size_y, f.stride_x, f.stride_y, f.padding_x, f.padding_y, wgs.c_str(), f.batch_normalize, f.groups); - netLayers.push_back(new tk::dnn::Conv2d(net, f.filters, f.size_x, f.size_y, f.stride_x, - f.stride_y, f.padding_x, f.padding_y, wgs, f.batch_normalize, false, f.groups)); + tk::dnn::Conv2d *l= new tk::dnn::Conv2d(net, f.filters, f.size_x, f.size_y, f.stride_x, + f.stride_y, f.padding_x, f.padding_y, wgs, f.batch_normalize, false, f.groups); if(f.activation != "linear") { tkdnnActivationMode_t act; if(f.activation == "relu") act = tkdnnActivationMode_t(CUDNN_ACTIVATION_RELU); else if(f.activation == "leaky") act = tk::dnn::ACTIVATION_LEAKY; else if(f.activation == "mish") act = tk::dnn::ACTIVATION_MISH; else { FatalError("activation not supported: " + f.activation); } - new tk::dnn::Activation(net, act); + netLayers.push_back(new tk::dnn::Activation(net, act)); + } else { + netLayers.push_back(l); } } else if(f.type == "shortcut") { if(f.layers.size() != 1) FatalError("no layers to shortcut\n"); diff --git a/tests/yolo3/yolo3.cpp b/tests/yolo3/yolo3.cpp index 70f0b9f..1036261 100644 --- a/tests/yolo3/yolo3.cpp +++ b/tests/yolo3/yolo3.cpp @@ -12,7 +12,6 @@ int main() { tk::dnn::Network *net = tk::dnn::darknetParser("../tests/yolo3/yolov3.cfg", "yolo3/layers"); net->print(); - std::vector yolo; for(int i=0; inum_layers; i++) { if(net->layers[i]->getLayerType() == tk::dnn::layerType_t::LAYER_YOLO) @@ -24,6 +23,10 @@ int main() { yolo[i]->classesNames = {"person" , "bicycle" , "car" , "motorbike" , "aeroplane" , "bus" , "train" , "truck" , "boat" , "traffic light" , "fire hydrant" , "stop sign" , "parking meter" , "bench" , "bird" , "cat" , "dog" , "horse" , "sheep" , "cow" , "elephant" , "bear" , "zebra" , "giraffe" , "backpack" , "umbrella" , "handbag" , "tie" , "suitcase" , "frisbee" , "skis" , "snowboard" , "sports ball" , "kite" , "baseball bat" , "baseball glove" , "skateboard" , "surfboard" , "tennis racket" , "bottle" , "wine glass" , "cup" , "fork" , "knife" , "spoon" , "bowl" , "banana" , "apple" , "sandwich" , "orange" , "broccoli" , "carrot" , "hot dog" , "pizza" , "donut" , "cake" , "chair" , "sofa" , "pottedplant" , "bed" , "diningtable" , "toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" , "cell phone" , "microwave" , "oven" , "toaster" , "sink" , "refrigerator" , "book" , "clock" , "vase" , "scissors" , "teddy bear" , "hair drier" , "toothbrush"}; } + //convert network to tensorRT + tk::dnn::NetworkRT netRT(net, net->getNetworkRTName("yolo3")); + + std::string input_bin = bin_path + "/layers/input.bin"; std::vector output_bins = { bin_path + "/debug/layer82_out.bin", @@ -36,16 +39,12 @@ int main() { dnnType *input_h; readBinaryFile(input_bin, net->input_dim.tot(), &input_h, &data); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(net, net->getNetworkRTName("yolo3")); - // the network have 3 outputs tk::dnn::dataDim_t out_dim[3]; for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; dnnType *cudnn_out[3], *rt_out[3]; - tk::dnn::dataDim_t dim1 = net->input_dim; //input dim + tk::dnn::dataDim_t dim1 = net->input_dim; //input dim printCenteredTitle(" CUDNN inference ", '=', 30); { dim1.print(); TIMER_START @@ -80,5 +79,4 @@ int main() { ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; } return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; - } -- 2.52.0 From a172492c8f7cda99fa97361a2f491620b5601185 Mon Sep 17 00:00:00 2001 From: Tane van der Boon <42484083+pullmyleg@users.noreply.github.com> Date: Mon, 1 Jun 2020 15:17:39 +1200 Subject: [PATCH 027/228] Update demo.cpp Added benchmark & save result command line parameters and showing average FPS at end of video result. --- demo/demo/demo.cpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 75012b6..b37ed7f 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -10,6 +10,7 @@ bool gRun; bool SAVE_RESULT = false; +bool BENCHMARK = false; void sig_handler(int signo) { std::cout<<"request gateway stop\n"; @@ -33,7 +34,13 @@ int main(int argc, char *argv[]) { ntype = argv[3][0]; int n_classes = 80; if(argc > 4) - n_classes = atoi(argv[4]); + n_classes = atoi(argv[4]); + if(argc > 5 && strcmp(argv[5], "benchmark") == 0) { + BENCHMARK = true; + } + if(argc > 5 && strcmp(argv[5], "save_result") == 0) { + SAVE_RESULT = true; + } tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; @@ -76,8 +83,9 @@ int main(int argc, char *argv[]) { cv::Mat frame; cv::Mat dnn_input; + if(!BENCHMARK) { cv::namedWindow("detection", cv::WINDOW_NORMAL); - + } std::vector detected_bbox; while(gRun) { @@ -91,9 +99,13 @@ int main(int argc, char *argv[]) { //inference detNN->update(dnn_input); + if(!BENCHMARK) { frame = detNN->draw(frame); + } + if(!BENCHMARK) { cv::imshow("detection", frame); + } cv::waitKey(1); if(SAVE_RESULT) resultVideo << frame; @@ -106,9 +118,9 @@ int main(int argc, char *argv[]) { std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n"; std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n"; for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); - std::cout<<"Avg: "< Date: Mon, 1 Jun 2020 15:33:05 +1200 Subject: [PATCH 028/228] Update README.md --- README.md | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 7f1016d..576a0f1 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,10 @@ M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Ti - [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101) - [3)Export weights for CenterNet](#3export-weights-for-centernet) - [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd) - - [Run the demo](#run-the-demo) + - [How to convert weights](#how-to-convert-weights) - [FP16 inference](#fp16-inference) - [INT8 inference](#int8-inference) + - [Run the demo](#run-the-demo) - [mAP demo](#map-demo) - [Existing tests and supported networks](#existing-tests-and-supported-networks) - [References](#references) @@ -113,26 +114,8 @@ cd pytorch-ssd conda env create -f env_mobv2ssd.yml python run_ssd_live_demo.py mb2-ssd-lite ``` -## Run the demo -To run the an object detection demo follow these steps (example with yolov3): -``` -rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files -./test_yolo3 # run the yolo test (is slow) -./demo yolo3_fp32.rt ../demo/yolo_test.mp4 y -``` -In general the demo program takes 4 parameters: -``` -./demo -``` -where -* `````` is the rt file generated by a test -* ```<``` is the path to a video file or a camera input -* `````` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family) -* ``````is the number of classes the network is trained on -N.b. By default it is used FP32 inference - -![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) +## How to convert weights ### FP16 inference @@ -187,6 +170,29 @@ rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT fil ./test_rtinference yolo3_fp32.rt 4 # test with a batch size of 4 ``` +## Run the demo + +To run the an object detection demo follow these steps (example with yolov3): +``` +rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files +./test_yolo3 # run the yolo test (is slow) +./demo yolo3_fp32.rt ../demo/yolo_test.mp4 y # add parameter y for yolo. c for CNET & m for Mobilenet. +./demo yolo3_fp32.rt ../demo/yolo_test.mp4 y 1 #if number of classes not 80. Add class number parameter. +``` +In general the demo program takes 5 parameters: +``` +./demo +``` +where +* `````` is the rt file generated by a test +* ```<``` is the path to a video file or a camera input +* `````` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family) +* ``````is the number of classes the network is trained on +N.b. By default it is used FP32 inference +* `````` benchmark or save_result. Adding benchmark will not show opencv detection video allowing demo to be run from command line and providing performance results without showing the video results. Adding save_result will save output of detection to results.mp4. + +![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) + ## mAP demo To compute mAP, precision, recall and f1score, run the map_demo. -- 2.52.0 From b7d2668de1d5b153cf8750f513d1265722a580a5 Mon Sep 17 00:00:00 2001 From: Tane van der Boon <42484083+pullmyleg@users.noreply.github.com> Date: Mon, 1 Jun 2020 15:35:13 +1200 Subject: [PATCH 029/228] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 576a0f1..5e46bd8 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ where * `````` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family) * ``````is the number of classes the network is trained on N.b. By default it is used FP32 inference -* `````` benchmark or save_result. Adding benchmark will not show opencv detection video allowing demo to be run from command line and providing performance results without showing the video results. Adding save_result will save output of detection to results.mp4. +* `````` benchmark or save_result. Adding benchmark will not show opencv detection video allowing demo to be run from terminal, providing performance results without showing the video. Adding save_result will save output of detection to results.mp4. ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) -- 2.52.0 From 826fcc97c8b38e890465ed02112b1e307fbace36 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 1 Jun 2020 10:34:48 +0200 Subject: [PATCH 030/228] Read classes' names from file Signed-off-by: Micaela Verucchi --- .gitignore | 2 + include/tkDNN/DarknetParser.h | 28 +++++++++--- tests/yolo3/coco.names | 80 +++++++++++++++++++++++++++++++++++ tests/yolo3/yolo3.cpp | 7 +-- 4 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 tests/yolo3/coco.names diff --git a/.gitignore b/.gitignore index 85a8b85..d62c96d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ build/ *.hdf5 *.pk *.table +demo/COCO_val2017 +demo/BDD100k_val \ No newline at end of file diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index 61040f9..0f0e12a 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -127,7 +127,7 @@ namespace tk { namespace dnn { } - void darknetAddLayer(tk::dnn::Network *net, darknetFields_t &f, std::string wgs_path, std::vector &netLayers) { + void darknetAddLayer(tk::dnn::Network *net, darknetFields_t &f, std::string wgs_path, std::vector &netLayers, const std::vector& names) { if(net == nullptr) FatalError("Cant add a layer without a Net\n"); @@ -180,14 +180,30 @@ namespace tk { namespace dnn { } else if(f.type == "yolo") { std::string wgs = wgs_path + "/g" + std::to_string(netLayers.size()) + ".bin"; printf("%d %d %s %d %f\n", f.classes, f.num/f.n_mask, wgs.c_str(), f.n_mask, f.scale_xy); - netLayers.push_back(new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy)); + tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy); + l->classesNames = names; + netLayers.push_back(l); } else{ FatalError("layer not supported: " + f.type); } } - tk::dnn::Network* darknetParser(std::string cfg_file, std::string wgs_path) { + std::vector darknetReadNames(const std::string& names_file){ + std::ifstream if_names(names_file); + if(!if_names.is_open()) + FatalError("cloud not open names file: " + names_file); + + std::vector names; + std::string line; + while(std::getline(if_names, line)) + names.push_back(line); + + if_names.close(); + return names; + } + + tk::dnn::Network* darknetParser(const std::string& cfg_file, const std::string& wgs_path, const std::string& names_file) { tk::dnn::Network *net = nullptr; @@ -198,6 +214,8 @@ namespace tk { namespace dnn { if(!if_cfg.is_open()) FatalError("cloud not open cfg file: " + cfg_file); + std::vector names = darknetReadNames(names_file); + darknetFields_t fields; // will be filled with layers fields std::string line; while(std::getline(if_cfg, line)) { @@ -218,7 +236,7 @@ namespace tk { namespace dnn { if(fields.type == "net") net = darknetAddNet(fields); else - darknetAddLayer(net, fields, wgs_path, netLayers); + darknetAddLayer(net, fields, wgs_path, netLayers, names); } // new type @@ -237,7 +255,7 @@ namespace tk { namespace dnn { // end of filled type if(fields.type != "") { - darknetAddLayer(net, fields, wgs_path, netLayers); + darknetAddLayer(net, fields, wgs_path, netLayers, names); } if(net == nullptr) { diff --git a/tests/yolo3/coco.names b/tests/yolo3/coco.names new file mode 100644 index 0000000..ca76c80 --- /dev/null +++ b/tests/yolo3/coco.names @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/tests/yolo3/yolo3.cpp b/tests/yolo3/yolo3.cpp index 1036261..89b8283 100644 --- a/tests/yolo3/yolo3.cpp +++ b/tests/yolo3/yolo3.cpp @@ -9,7 +9,7 @@ int main() { std::string bin_path = "yolo3"; downloadWeightsifDoNotExist("yolo3/layers/input.bin", bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); - tk::dnn::Network *net = tk::dnn::darknetParser("../tests/yolo3/yolov3.cfg", "yolo3/layers"); + tk::dnn::Network *net = tk::dnn::darknetParser("../tests/yolo3/yolov3.cfg", "yolo3/layers", "../tests/yolo3/coco.names"); net->print(); std::vector yolo; @@ -18,11 +18,6 @@ int main() { yolo.push_back((tk::dnn::Yolo*)net->layers[i]); } - // fill classes names - for(int i=0; i<3; i++) { - yolo[i]->classesNames = {"person" , "bicycle" , "car" , "motorbike" , "aeroplane" , "bus" , "train" , "truck" , "boat" , "traffic light" , "fire hydrant" , "stop sign" , "parking meter" , "bench" , "bird" , "cat" , "dog" , "horse" , "sheep" , "cow" , "elephant" , "bear" , "zebra" , "giraffe" , "backpack" , "umbrella" , "handbag" , "tie" , "suitcase" , "frisbee" , "skis" , "snowboard" , "sports ball" , "kite" , "baseball bat" , "baseball glove" , "skateboard" , "surfboard" , "tennis racket" , "bottle" , "wine glass" , "cup" , "fork" , "knife" , "spoon" , "bowl" , "banana" , "apple" , "sandwich" , "orange" , "broccoli" , "carrot" , "hot dog" , "pizza" , "donut" , "cake" , "chair" , "sofa" , "pottedplant" , "bed" , "diningtable" , "toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" , "cell phone" , "microwave" , "oven" , "toaster" , "sink" , "refrigerator" , "book" , "clock" , "vase" , "scissors" , "teddy bear" , "hair drier" , "toothbrush"}; - } - //convert network to tensorRT tk::dnn::NetworkRT netRT(net, net->getNetworkRTName("yolo3")); -- 2.52.0 From a6eef498fabf1ce09a1d67fe61b91f03561ad7de Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 1 Jun 2020 10:39:13 +0200 Subject: [PATCH 031/228] Check Signed-off-by: Micaela Verucchi --- include/tkDNN/DarknetParser.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index 0f0e12a..9008670 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -181,6 +181,8 @@ namespace tk { namespace dnn { std::string wgs = wgs_path + "/g" + std::to_string(netLayers.size()) + ".bin"; printf("%d %d %s %d %f\n", f.classes, f.num/f.n_mask, wgs.c_str(), f.n_mask, f.scale_xy); tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy); + if(names.size() != f.classes) + FatalError("Mismatch between number of classes and names"); l->classesNames = names; netLayers.push_back(l); @@ -197,7 +199,8 @@ namespace tk { namespace dnn { std::vector names; std::string line; while(std::getline(if_names, line)) - names.push_back(line); + if(line != "") + names.push_back(line); if_names.close(); return names; -- 2.52.0 From d8fbee58d8e417ca6649c9b216fe388b2d5c4ed7 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 1 Jun 2020 10:42:06 +0200 Subject: [PATCH 032/228] Add names files Signed-off-by: Micaela Verucchi --- tests/yolo3/berkeley.names | 10 ++++++++++ tests/yolo3/voc.names | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tests/yolo3/berkeley.names create mode 100644 tests/yolo3/voc.names diff --git a/tests/yolo3/berkeley.names b/tests/yolo3/berkeley.names new file mode 100644 index 0000000..321e633 --- /dev/null +++ b/tests/yolo3/berkeley.names @@ -0,0 +1,10 @@ +person +car +truck +bus +motor +bike +rider +traffic light +traffic sign +train \ No newline at end of file diff --git a/tests/yolo3/voc.names b/tests/yolo3/voc.names new file mode 100644 index 0000000..8420ab3 --- /dev/null +++ b/tests/yolo3/voc.names @@ -0,0 +1,20 @@ +aeroplane +bicycle +bird +boat +bottle +bus +car +cat +chair +cow +diningtable +dog +horse +motorbike +person +pottedplant +sheep +sofa +train +tvmonitor -- 2.52.0 From d2e2669b6d9170bd3c68a11cfd432d22afd315fc Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Mon, 1 Jun 2020 12:22:55 +0200 Subject: [PATCH 033/228] darknet parse all net to be tested --- CMakeLists.txt | 130 +--- include/tkDNN/models/Yolo3.h | 289 -------- include/tkDNN/test.h | 68 ++ scripts/test_all_tests.sh | 10 +- src/Network.cpp | 3 +- src/NetworkRT.cpp | 2 +- src/Yolo.cpp | 3 +- tests/{ => backbones}/dla34/dla34.cpp | 0 .../dla34/dla34_weightsexporter.py | 0 tests/{ => backbones}/dla34/env_dla34.yml | 0 .../resnet101/env_resnet101.yml | 0 tests/{ => backbones}/resnet101/resnet101.cpp | 0 .../resnet101/resnet101_weightsexporter.py | 0 .../bdd-csresnext50-panet-spp.cpp | 554 --------------- tests/build_models.sh | 18 - .../{ => centernet}/dla34_cnet/dla34_cnet.cpp | 0 .../resnet101_cnet/resnet101_cnet.cpp | 0 .../csresnext50-panet-spp.cpp | 554 --------------- .../cfg}/csresnext50-panet-spp.cfg | 0 .../cfg/csresnext50-panet-spp_berkeley.cfg} | 0 .../{yolo/yolo.cfg => darknet/cfg/yolo2.cfg} | 0 .../cfg/yolo2_voc.cfg} | 0 .../cfg/yolo2tiny.cfg} | 0 .../yolov3.cfg => darknet/cfg/yolo3.cfg} | 0 .../cfg/yolo3_512.cfg} | 28 +- .../cfg}/yolo3_berkeley.cfg | 0 .../cfg/yolo3_coco4.cfg} | 0 .../cfg}/yolo3_flir.cfg | 0 .../cfg/yolo3tiny.cfg} | 0 .../cfg/yolo3tiny_512.cfg} | 12 +- .../yolov4.cfg => darknet/cfg/yolo4.cfg} | 0 tests/darknet/csresnext50-panet-spp.cpp | 33 + .../csresnext50-panet-spp_berkeley.cpp | 33 + tests/{yolo3 => darknet/names}/berkeley.names | 0 tests/{yolo3 => darknet/names}/coco.names | 0 tests/darknet/names/coco4.names | 4 + tests/darknet/names/flir.names | 3 + tests/{yolo3 => darknet/names}/voc.names | 0 tests/darknet/yolo2.cpp | 31 + tests/darknet/yolo2_voc.cpp | 32 + tests/darknet/yolo2tiny.cpp | 31 + tests/darknet/yolo3.cpp | 33 + tests/darknet/yolo3_512.cpp | 33 + tests/darknet/yolo3_berkeley.cpp | 33 + tests/darknet/yolo3_coco4.cpp | 33 + tests/darknet/yolo3_flir.cpp | 33 + tests/darknet/yolo3tiny.cpp | 31 + tests/darknet/yolo3tiny512.cpp | 31 + tests/darknet/yolo4.cpp | 33 + .../{ => exporters}/caffe_weights_exporter.py | 0 .../keras_weights_exporter.py} | 0 .../bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp | 0 .../mobilenetv2ssd/mobilenetv2ssd.cpp | 0 .../mobilenetv2ssd512/mobilenetv2ssd512.cpp | 0 tests/yolo/yolo.cpp | 157 ----- tests/yolo3/yolo3.cpp | 77 -- tests/yolo3_512/yolo3_512.cpp | 99 --- tests/yolo3_512tp/yolo3_512tp.cpp | 97 --- tests/yolo3_berkeley/yolo3_berkeley.cpp | 99 --- tests/yolo3_coco4/yolo3_coco4.cpp | 97 --- tests/yolo3_flir/yolo3_flir.cpp | 100 --- tests/yolo3_tiny/yolo3_tiny.cpp | 130 ---- tests/yolo3_tiny512/yolo3_tiny512.cpp | 128 ---- tests/yolo3_tiny512tp/yolo3_tiny512tp.cpp | 127 ---- tests/yolo3_tinyNM512/yolo3_tinyNM512.cpp | 127 ---- tests/yolo4/yolo4.cpp | 666 ------------------ tests/yolo_224/yolo_224.cfg | 258 ------- tests/yolo_224/yolo_224.cpp | 156 ---- tests/yolo_berkeley/yolo_berkeley.cpp | 156 ---- .../yolov2-voc-10-resize-test.cfg | 259 ------- tests/yolo_relu/yolo_relu.cfg | 258 ------- tests/yolo_relu/yolo_relu.cpp | 155 ---- tests/yolo_tiny/yolo_tiny.cpp | 100 --- tests/yolo_voc/yolo_voc.cpp | 156 ---- 74 files changed, 560 insertions(+), 4940 deletions(-) delete mode 100644 include/tkDNN/models/Yolo3.h create mode 100644 include/tkDNN/test.h rename tests/{ => backbones}/dla34/dla34.cpp (100%) rename tests/{ => backbones}/dla34/dla34_weightsexporter.py (100%) rename tests/{ => backbones}/dla34/env_dla34.yml (100%) rename tests/{ => backbones}/resnet101/env_resnet101.yml (100%) rename tests/{ => backbones}/resnet101/resnet101.cpp (100%) rename tests/{ => backbones}/resnet101/resnet101_weightsexporter.py (100%) delete mode 100644 tests/bdd-csresnext50-panet-spp/bdd-csresnext50-panet-spp.cpp delete mode 100644 tests/build_models.sh rename tests/{ => centernet}/dla34_cnet/dla34_cnet.cpp (100%) rename tests/{ => centernet}/resnet101_cnet/resnet101_cnet.cpp (100%) delete mode 100644 tests/csresnext50-panet-spp/csresnext50-panet-spp.cpp rename tests/{csresnext50-panet-spp => darknet/cfg}/csresnext50-panet-spp.cfg (100%) rename tests/{bdd-csresnext50-panet-spp/berkeleycsresnetx50.cfg => darknet/cfg/csresnext50-panet-spp_berkeley.cfg} (100%) rename tests/{yolo/yolo.cfg => darknet/cfg/yolo2.cfg} (100%) rename tests/{yolo_voc/yolo_voc.cfg => darknet/cfg/yolo2_voc.cfg} (100%) rename tests/{yolo_tiny/tiny-yolo.cfg => darknet/cfg/yolo2tiny.cfg} (100%) rename tests/{yolo3/yolov3.cfg => darknet/cfg/yolo3.cfg} (100%) rename tests/{yolo3_512tp/yolo3512.cfg => darknet/cfg/yolo3_512.cfg} (93%) rename tests/{yolo3_berkeley => darknet/cfg}/yolo3_berkeley.cfg (100%) rename tests/{yolo3_coco4/yolov3-coco4.cfg => darknet/cfg/yolo3_coco4.cfg} (100%) rename tests/{yolo3_flir => darknet/cfg}/yolo3_flir.cfg (100%) rename tests/{yolo3_tiny/yolov3-tiny.cfg => darknet/cfg/yolo3tiny.cfg} (100%) rename tests/{yolo3_tiny512tp/yolo3tiny512.cfg => darknet/cfg/yolo3tiny_512.cfg} (87%) rename tests/{yolo4/yolov4.cfg => darknet/cfg/yolo4.cfg} (100%) create mode 100644 tests/darknet/csresnext50-panet-spp.cpp create mode 100644 tests/darknet/csresnext50-panet-spp_berkeley.cpp rename tests/{yolo3 => darknet/names}/berkeley.names (100%) rename tests/{yolo3 => darknet/names}/coco.names (100%) create mode 100644 tests/darknet/names/coco4.names create mode 100644 tests/darknet/names/flir.names rename tests/{yolo3 => darknet/names}/voc.names (100%) create mode 100644 tests/darknet/yolo2.cpp create mode 100644 tests/darknet/yolo2_voc.cpp create mode 100644 tests/darknet/yolo2tiny.cpp create mode 100644 tests/darknet/yolo3.cpp create mode 100644 tests/darknet/yolo3_512.cpp create mode 100644 tests/darknet/yolo3_berkeley.cpp create mode 100644 tests/darknet/yolo3_coco4.cpp create mode 100644 tests/darknet/yolo3_flir.cpp create mode 100644 tests/darknet/yolo3tiny.cpp create mode 100644 tests/darknet/yolo3tiny512.cpp create mode 100644 tests/darknet/yolo4.cpp rename tests/{ => exporters}/caffe_weights_exporter.py (100%) rename tests/{weights_exporter.py => exporters/keras_weights_exporter.py} (100%) rename tests/{ => mobilenet}/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp (100%) rename tests/{ => mobilenet}/mobilenetv2ssd/mobilenetv2ssd.cpp (100%) rename tests/{ => mobilenet}/mobilenetv2ssd512/mobilenetv2ssd512.cpp (100%) delete mode 100644 tests/yolo/yolo.cpp delete mode 100644 tests/yolo3/yolo3.cpp delete mode 100644 tests/yolo3_512/yolo3_512.cpp delete mode 100644 tests/yolo3_512tp/yolo3_512tp.cpp delete mode 100644 tests/yolo3_berkeley/yolo3_berkeley.cpp delete mode 100644 tests/yolo3_coco4/yolo3_coco4.cpp delete mode 100644 tests/yolo3_flir/yolo3_flir.cpp delete mode 100644 tests/yolo3_tiny/yolo3_tiny.cpp delete mode 100644 tests/yolo3_tiny512/yolo3_tiny512.cpp delete mode 100644 tests/yolo3_tiny512tp/yolo3_tiny512tp.cpp delete mode 100644 tests/yolo3_tinyNM512/yolo3_tinyNM512.cpp delete mode 100644 tests/yolo4/yolo4.cpp delete mode 100644 tests/yolo_224/yolo_224.cfg delete mode 100644 tests/yolo_224/yolo_224.cpp delete mode 100644 tests/yolo_berkeley/yolo_berkeley.cpp delete mode 100644 tests/yolo_berkeley/yolov2-voc-10-resize-test.cfg delete mode 100644 tests/yolo_relu/yolo_relu.cfg delete mode 100644 tests/yolo_relu/yolo_relu.cpp delete mode 100644 tests/yolo_tiny/yolo_tiny.cpp delete mode 100644 tests/yolo_voc/yolo_voc.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 77da651..376c175 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,7 @@ target_link_libraries(tkDNN ${tkdnn_LIBS}) #add_library(tkDNN_static STATIC ${tkdnn_SRC}) #target_link_libraries(tkDNN_static ${tkdnn_LIBS}) +# SMALL NETS add_executable(test_simple tests/simple/test_simple.cpp) target_link_libraries(test_simple tkDNN) @@ -63,91 +64,43 @@ target_link_libraries(test_mnist tkDNN) add_executable(test_mnistRT tests/mnist/test_mnistRT.cpp) target_link_libraries(test_mnistRT tkDNN) -## YOLO NETS -add_executable(test_yolo tests/yolo/yolo.cpp) -target_link_libraries(test_yolo tkDNN) - -add_executable(test_yolo_voc tests/yolo_voc/yolo_voc.cpp) -target_link_libraries(test_yolo_voc tkDNN) - -add_executable(test_yolo_tiny tests/yolo_tiny/yolo_tiny.cpp) -target_link_libraries(test_yolo_tiny tkDNN) - -add_executable(test_yolo_relu tests/yolo_relu/yolo_relu.cpp) -target_link_libraries(test_yolo_relu tkDNN) - - -add_executable(test_yolo_224 tests/yolo_224/yolo_224.cpp) -target_link_libraries(test_yolo_224 tkDNN) - -add_executable(test_yolo_berkeley tests/yolo_berkeley/yolo_berkeley.cpp) -target_link_libraries(test_yolo_berkeley tkDNN) - -add_executable(test_yolo3_coco4 tests/yolo3_coco4/yolo3_coco4.cpp) -target_link_libraries(test_yolo3_coco4 tkDNN) - -add_executable(test_yolo3 tests/yolo3/yolo3.cpp) -target_link_libraries(test_yolo3 tkDNN) - -add_executable(test_yolo3_512 tests/yolo3_512/yolo3_512.cpp) -target_link_libraries(test_yolo3_512 tkDNN) - -add_executable(test_yolo3_512tp tests/yolo3_512tp/yolo3_512tp.cpp) -target_link_libraries(test_yolo3_512tp tkDNN) - -add_executable(test_yolo3_tiny tests/yolo3_tiny/yolo3_tiny.cpp) -target_link_libraries(test_yolo3_tiny tkDNN) - -add_executable(test_yolo3_tiny512 tests/yolo3_tiny512/yolo3_tiny512.cpp) -target_link_libraries(test_yolo3_tiny512 tkDNN) - -add_executable(test_yolo3_tinyNM512 tests/yolo3_tinyNM512/yolo3_tinyNM512.cpp) -target_link_libraries(test_yolo3_tinyNM512 tkDNN) - -add_executable(test_yolo3_tiny512tp tests/yolo3_tiny512tp/yolo3_tiny512tp.cpp) -target_link_libraries(test_yolo3_tiny512tp tkDNN) - -add_executable(test_yolo3_berkeley tests/yolo3_berkeley/yolo3_berkeley.cpp) -target_link_libraries(test_yolo3_berkeley tkDNN) - -add_executable(test_yolo3_flir tests/yolo3_flir/yolo3_flir.cpp) -target_link_libraries(test_yolo3_flir tkDNN) - -add_executable(test_yolo4 tests/yolo4/yolo4.cpp) -target_link_libraries(test_yolo4 tkDNN) - -add_executable(test_mobilenetv2ssd tests/mobilenetv2ssd/mobilenetv2ssd.cpp) -target_link_libraries(test_mobilenetv2ssd tkDNN) - -add_executable(test_bdd-mobilenetv2ssd tests/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp) -target_link_libraries(test_bdd-mobilenetv2ssd tkDNN) - -add_executable(test_mobilenetv2ssd512 tests/mobilenetv2ssd512/mobilenetv2ssd512.cpp) -target_link_libraries(test_mobilenetv2ssd512 tkDNN) - -add_executable(test_resnet101 tests/resnet101/resnet101.cpp) -target_link_libraries(test_resnet101 tkDNN) - -add_executable(test_csresnext50-panet-spp tests/csresnext50-panet-spp/csresnext50-panet-spp.cpp) -target_link_libraries(test_csresnext50-panet-spp tkDNN) - -add_executable(test_bdd-csresnext50-panet-spp tests/bdd-csresnext50-panet-spp/bdd-csresnext50-panet-spp.cpp) -target_link_libraries(test_bdd-csresnext50-panet-spp tkDNN) - -add_executable(test_resnet101_cnet tests/resnet101_cnet/resnet101_cnet.cpp) -target_link_libraries(test_resnet101_cnet tkDNN) - -add_executable(test_dla34 tests/dla34/dla34.cpp) -target_link_libraries(test_dla34 tkDNN) - -add_executable(test_dla34_cnet tests/dla34_cnet/dla34_cnet.cpp) -target_link_libraries(test_dla34_cnet tkDNN) - add_executable(test_imuodom tests/imuodom/imuodom.cpp) target_link_libraries(test_imuodom tkDNN) -################################################################################ +# DARKNET +file(GLOB darknet_SRC "tests/darknet/*.cpp") +foreach(test_SRC ${darknet_SRC}) + get_filename_component(test_NAME "${test_SRC}" NAME_WE) + set(test_NAME test_${test_NAME}) + add_executable(${test_NAME} ${test_SRC}) + target_link_libraries(${test_NAME} tkDNN) +endforeach() +# MOBILENET +add_executable(test_mobilenetv2ssd tests/mobilenet/mobilenetv2ssd/mobilenetv2ssd.cpp) +target_link_libraries(test_mobilenetv2ssd tkDNN) + +add_executable(test_bdd-mobilenetv2ssd tests/mobilenet/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp) +target_link_libraries(test_bdd-mobilenetv2ssd tkDNN) + +add_executable(test_mobilenetv2ssd512 tests/mobilenet/mobilenetv2ssd512/mobilenetv2ssd512.cpp) +target_link_libraries(test_mobilenetv2ssd512 tkDNN) + +# BACKBONES +add_executable(test_resnet101 tests/backbones/resnet101/resnet101.cpp) +target_link_libraries(test_resnet101 tkDNN) + +add_executable(test_dla34 tests/backbones/dla34/dla34.cpp) +target_link_libraries(test_dla34 tkDNN) + +# CENTERNET +add_executable(test_resnet101_cnet tests/centernet/resnet101_cnet/resnet101_cnet.cpp) +target_link_libraries(test_resnet101_cnet tkDNN) + +add_executable(test_dla34_cnet tests/centernet/dla34_cnet/dla34_cnet.cpp) +target_link_libraries(test_dla34_cnet tkDNN) + +# DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) @@ -171,18 +124,3 @@ install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/cmake/" # source directory DESTINATION "share/tkDNN/cmake/" # target directory ) - -#------------------------------------------------------------------------------- -# Prepare for test (not needed anymore) -#------------------------------------------------------------------------------- -#set(TEST_DATA true CACHE BOOL "If true download deps") -#if( ${TEST_DATA} ) -# message("Launching pre-build dependency installer script...") -# -# execute_process (COMMAND bash -c "bash build_models.sh download" -# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests) -# -# set(TEST_DATA false CACHE BOOL "If true download deps" FORCE) -# message("Finished dowloading test weights") -#endif() - diff --git a/include/tkDNN/models/Yolo3.h b/include/tkDNN/models/Yolo3.h deleted file mode 100644 index cd69b32..0000000 --- a/include/tkDNN/models/Yolo3.h +++ /dev/null @@ -1,289 +0,0 @@ -int preYoloFilters = (classes+5)*3; - -std::string input_bin = bin_path + "/layers/input.bin"; -std::vector output_bins = { - bin_path + "/debug/layer82_out.bin", - bin_path + "/debug/layer94_out.bin", - bin_path + "/debug/layer106_out.bin" -}; -std::string c0_bin = bin_path + "/layers/c0.bin"; -std::string c1_bin = bin_path + "/layers/c1.bin"; -std::string c2_bin = bin_path + "/layers/c2.bin"; -std::string c3_bin = bin_path + "/layers/c3.bin"; -std::string c5_bin = bin_path + "/layers/c5.bin"; -std::string c6_bin = bin_path + "/layers/c6.bin"; -std::string c7_bin = bin_path + "/layers/c7.bin"; -std::string c9_bin = bin_path + "/layers/c9.bin"; -std::string c10_bin = bin_path + "/layers/c10.bin"; -std::string c12_bin = bin_path + "/layers/c12.bin"; -std::string c13_bin = bin_path + "/layers/c13.bin"; -std::string c14_bin = bin_path + "/layers/c14.bin"; -std::string c16_bin = bin_path + "/layers/c16.bin"; -std::string c17_bin = bin_path + "/layers/c17.bin"; -std::string c19_bin = bin_path + "/layers/c19.bin"; -std::string c20_bin = bin_path + "/layers/c20.bin"; -std::string c22_bin = bin_path + "/layers/c22.bin"; -std::string c23_bin = bin_path + "/layers/c23.bin"; -std::string c25_bin = bin_path + "/layers/c25.bin"; -std::string c26_bin = bin_path + "/layers/c26.bin"; -std::string c28_bin = bin_path + "/layers/c28.bin"; -std::string c29_bin = bin_path + "/layers/c29.bin"; -std::string c31_bin = bin_path + "/layers/c31.bin"; -std::string c32_bin = bin_path + "/layers/c32.bin"; -std::string c34_bin = bin_path + "/layers/c34.bin"; -std::string c35_bin = bin_path + "/layers/c35.bin"; -std::string c37_bin = bin_path + "/layers/c37.bin"; -std::string c38_bin = bin_path + "/layers/c38.bin"; -std::string c39_bin = bin_path + "/layers/c39.bin"; -std::string c41_bin = bin_path + "/layers/c41.bin"; -std::string c42_bin = bin_path + "/layers/c42.bin"; -std::string c44_bin = bin_path + "/layers/c44.bin"; -std::string c45_bin = bin_path + "/layers/c45.bin"; -std::string c47_bin = bin_path + "/layers/c47.bin"; -std::string c48_bin = bin_path + "/layers/c48.bin"; -std::string c50_bin = bin_path + "/layers/c50.bin"; -std::string c51_bin = bin_path + "/layers/c51.bin"; -std::string c53_bin = bin_path + "/layers/c53.bin"; -std::string c54_bin = bin_path + "/layers/c54.bin"; -std::string c56_bin = bin_path + "/layers/c56.bin"; -std::string c57_bin = bin_path + "/layers/c57.bin"; -std::string c59_bin = bin_path + "/layers/c59.bin"; -std::string c60_bin = bin_path + "/layers/c60.bin"; -std::string c62_bin = bin_path + "/layers/c62.bin"; -std::string c63_bin = bin_path + "/layers/c63.bin"; -std::string c64_bin = bin_path + "/layers/c64.bin"; -std::string c66_bin = bin_path + "/layers/c66.bin"; -std::string c67_bin = bin_path + "/layers/c67.bin"; -std::string c69_bin = bin_path + "/layers/c69.bin"; -std::string c70_bin = bin_path + "/layers/c70.bin"; -std::string c72_bin = bin_path + "/layers/c72.bin"; -std::string c73_bin = bin_path + "/layers/c73.bin"; -std::string c75_bin = bin_path + "/layers/c75.bin"; -std::string c76_bin = bin_path + "/layers/c76.bin"; -std::string c77_bin = bin_path + "/layers/c77.bin"; -std::string c78_bin = bin_path + "/layers/c78.bin"; -std::string c79_bin = bin_path + "/layers/c79.bin"; -std::string c80_bin = bin_path + "/layers/c80.bin"; -std::string c81_bin = bin_path + "/layers/c81.bin"; -std::string g82_bin = bin_path + "/layers/g82.bin"; -std::string c84_bin = bin_path + "/layers/c84.bin"; -std::string c87_bin = bin_path + "/layers/c87.bin"; -std::string c88_bin = bin_path + "/layers/c88.bin"; -std::string c89_bin = bin_path + "/layers/c89.bin"; -std::string c90_bin = bin_path + "/layers/c90.bin"; -std::string c91_bin = bin_path + "/layers/c91.bin"; -std::string c92_bin = bin_path + "/layers/c92.bin"; -std::string c93_bin = bin_path + "/layers/c93.bin"; -std::string g94_bin = bin_path + "/layers/g94.bin"; -std::string c96_bin = bin_path + "/layers/c96.bin"; -std::string c99_bin = bin_path + "/layers/c99.bin"; -std::string c100_bin = bin_path + "/layers/c100.bin"; -std::string c101_bin = bin_path + "/layers/c101.bin"; -std::string c102_bin = bin_path + "/layers/c102.bin"; -std::string c103_bin = bin_path + "/layers/c103.bin"; -std::string c104_bin = bin_path + "/layers/c104.bin"; -std::string c105_bin = bin_path + "/layers/c105.bin"; -std::string g106_bin = bin_path + "/layers/g106.bin"; - -tk::dnn::Conv2d c0 (&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); -tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c1 (&net, 64, 3, 3, 2, 2, 1, 1, c1_bin, true); -tk::dnn::Activation a1 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c2 (&net, 32, 1, 1, 1, 1, 0, 0, c2_bin, true); -tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c3 (&net, 64, 3, 3, 1, 1, 1, 1, c3_bin, true); -tk::dnn::Activation a3 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s4 (&net, &a1); -tk::dnn::Conv2d c5 (&net, 128, 3, 3, 2, 2, 1, 1, c5_bin, true); -tk::dnn::Activation a5 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c6 (&net, 64, 1, 1, 1, 1, 0, 0, c6_bin, true); -tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c7 (&net, 128, 3, 3, 1, 1, 1, 1, c7_bin, true); -tk::dnn::Activation a7 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s8 (&net, &a5); -tk::dnn::Conv2d c9 (&net, 64, 1, 1, 1, 1, 0, 0, c9_bin, true); -tk::dnn::Activation a9 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c10 (&net, 128, 3, 3, 1, 1, 1, 1, c10_bin, true); -tk::dnn::Activation a10 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s11 (&net, &s8); - -tk::dnn::Conv2d c12 (&net, 256, 3, 3, 2, 2, 1, 1, c12_bin, true); -tk::dnn::Activation a12 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c13 (&net, 128, 1, 1, 1, 1, 0, 0, c13_bin, true); -tk::dnn::Activation a13 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c14 (&net, 256, 3, 3, 1, 1, 1, 1, c14_bin, true); -tk::dnn::Activation a14 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s15 (&net, &a12); - -tk::dnn::Conv2d c16 (&net, 128, 1, 1, 1, 1, 0, 0, c16_bin, true); -tk::dnn::Activation a16 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c17 (&net, 256, 3, 3, 1, 1, 1, 1, c17_bin, true); -tk::dnn::Activation a17 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s18 (&net, &s15); -tk::dnn::Conv2d c19 (&net, 128, 1, 1, 1, 1, 0, 0, c19_bin, true); -tk::dnn::Activation a19 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c20 (&net, 256, 3, 3, 1, 1, 1, 1, c20_bin, true); -tk::dnn::Activation a20 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s21 (&net, &s18); -tk::dnn::Conv2d c22 (&net, 128, 1, 1, 1, 1, 0, 0, c22_bin, true); -tk::dnn::Activation a22 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c23 (&net, 256, 3, 3, 1, 1, 1, 1, c23_bin, true); -tk::dnn::Activation a23 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s24 (&net, &s21); -tk::dnn::Conv2d c25 (&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true); -tk::dnn::Activation a25 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c26 (&net, 256, 3, 3, 1, 1, 1, 1, c26_bin, true); -tk::dnn::Activation a26 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s27 (&net, &s24); -tk::dnn::Conv2d c28 (&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true); -tk::dnn::Activation a28 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c29 (&net, 256, 3, 3, 1, 1, 1, 1, c29_bin, true); -tk::dnn::Activation a29 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s30 (&net, &s27); -tk::dnn::Conv2d c31 (&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true); -tk::dnn::Activation a31 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c32 (&net, 256, 3, 3, 1, 1, 1, 1, c32_bin, true); -tk::dnn::Activation a32 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s33 (&net, &s30); -tk::dnn::Conv2d c34 (&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true); -tk::dnn::Activation a34 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c35 (&net, 256, 3, 3, 1, 1, 1, 1, c35_bin, true); -tk::dnn::Activation a35 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s36 (&net, &s33); - -tk::dnn::Conv2d c37 (&net, 512, 3, 3, 2, 2, 1, 1, c37_bin, true); -tk::dnn::Activation a37 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c38 (&net, 256, 1, 1, 1, 1, 0, 0, c38_bin, true); -tk::dnn::Activation a38 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c39 (&net, 512, 3, 3, 1, 1, 1, 1, c39_bin, true); -tk::dnn::Activation a39 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s40 (&net, &a37); - -tk::dnn::Conv2d c41 (&net, 256, 1, 1, 1, 1, 0, 0, c41_bin, true); -tk::dnn::Activation a41 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c42 (&net, 512, 3, 3, 1, 1, 1, 1, c42_bin, true); -tk::dnn::Activation a42 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s43 (&net, &s40); -tk::dnn::Conv2d c44 (&net, 256, 1, 1, 1, 1, 0, 0, c44_bin, true); -tk::dnn::Activation a44 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c45 (&net, 512, 3, 3, 1, 1, 1, 1, c45_bin, true); -tk::dnn::Activation a45 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s46 (&net, &s43); -tk::dnn::Conv2d c47 (&net, 256, 1, 1, 1, 1, 0, 0, c47_bin, true); -tk::dnn::Activation a47 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c48 (&net, 512, 3, 3, 1, 1, 1, 1, c48_bin, true); -tk::dnn::Activation a48 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s49 (&net, &s46); -tk::dnn::Conv2d c50 (&net, 256, 1, 1, 1, 1, 0, 0, c50_bin, true); -tk::dnn::Activation a50 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c51 (&net, 512, 3, 3, 1, 1, 1, 1, c51_bin, true); -tk::dnn::Activation a51 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s52 (&net, &s49); -tk::dnn::Conv2d c53 (&net, 256, 1, 1, 1, 1, 0, 0, c53_bin, true); -tk::dnn::Activation a53 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c54 (&net, 512, 3, 3, 1, 1, 1, 1, c54_bin, true); -tk::dnn::Activation a54 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s55 (&net, &s52); -tk::dnn::Conv2d c56 (&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true); -tk::dnn::Activation a56 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c57 (&net, 512, 3, 3, 1, 1, 1, 1, c57_bin, true); -tk::dnn::Activation a57 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s58 (&net, &s55); -tk::dnn::Conv2d c59 (&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true); -tk::dnn::Activation a59 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c60 (&net, 512, 3, 3, 1, 1, 1, 1, c60_bin, true); -tk::dnn::Activation a60 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s61 (&net, &s58); - -tk::dnn::Conv2d c62 (&net,1024, 3, 3, 2, 2, 1, 1, c62_bin, true); -tk::dnn::Activation a62 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c63 (&net, 512, 1, 1, 1, 1, 0, 0, c63_bin, true); -tk::dnn::Activation a63 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c64 (&net,1024, 3, 3, 1, 1, 1, 1, c64_bin, true); -tk::dnn::Activation a64 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s65 (&net, &a62); - -tk::dnn::Conv2d c66 (&net, 512, 1, 1, 1, 1, 0, 0, c66_bin, true); -tk::dnn::Activation a66 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c67 (&net,1024, 3, 3, 1, 1, 1, 1, c67_bin, true); -tk::dnn::Activation a67 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s68 (&net, &s65); - -tk::dnn::Conv2d c69 (&net, 512, 1, 1, 1, 1, 0, 0, c69_bin, true); -tk::dnn::Activation a69 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c70 (&net,1024, 3, 3, 1, 1, 1, 1, c70_bin, true); -tk::dnn::Activation a70 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s71 (&net, &s68); - -tk::dnn::Conv2d c72 (&net, 512, 1, 1, 1, 1, 0, 0, c72_bin, true); -tk::dnn::Activation a72 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c73 (&net,1024, 3, 3, 1, 1, 1, 1, c73_bin, true); -tk::dnn::Activation a73 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Shortcut s74 (&net, &s71); - -tk::dnn::Conv2d c75 (&net, 512, 1, 1, 1, 1, 0, 0, c75_bin, true); -tk::dnn::Activation a75 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c76 (&net,1024, 3, 3, 1, 1, 1, 1, c76_bin, true); -tk::dnn::Activation a76 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c77 (&net, 512, 1, 1, 1, 1, 0, 0, c77_bin, true); -tk::dnn::Activation a77 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c78 (&net,1024, 3, 3, 1, 1, 1, 1, c78_bin, true); -tk::dnn::Activation a78 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c79 (&net, 512, 1, 1, 1, 1, 0, 0, c79_bin, true); -tk::dnn::Activation a79 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c80 (&net,1024, 3, 3, 1, 1, 1, 1, c80_bin, true); -tk::dnn::Activation a80 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c81 (&net, preYoloFilters, 1, 1, 1, 1, 0, 0, c81_bin, false); -tk::dnn::Yolo yolo0 (&net, classes, 3, g82_bin); - -tk::dnn::Layer *m83_layers[1] = { &a79 }; -tk::dnn::Route m83 (&net, m83_layers, 1); -tk::dnn::Conv2d c84 (&net, 256, 1, 1, 1, 1, 0, 0, c84_bin, true); -tk::dnn::Activation a84 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Upsample u85 (&net, 2); - -tk::dnn::Layer *m86_layers[2] = { &u85, &s61 }; -tk::dnn::Route m86 (&net, m86_layers, 2); -tk::dnn::Conv2d c87 (&net, 256, 1, 1, 1, 1, 0, 0, c87_bin, true); -tk::dnn::Activation a87 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c88 (&net, 512, 3, 3, 1, 1, 1, 1, c88_bin, true); -tk::dnn::Activation a88 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c89 (&net, 256, 1, 1, 1, 1, 0, 0, c89_bin, true); -tk::dnn::Activation a89 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c90 (&net, 512, 3, 3, 1, 1, 1, 1, c90_bin, true); -tk::dnn::Activation a90 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c91 (&net, 256, 1, 1, 1, 1, 0, 0, c91_bin, true); -tk::dnn::Activation a91 (&net, tk::dnn::ACTIVATION_LEAKY); - -tk::dnn::Conv2d c92 (&net, 512, 3, 3, 1, 1, 1, 1, c92_bin, true); -tk::dnn::Activation a92 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c93 (&net, preYoloFilters, 1, 1, 1, 1, 0, 0, c93_bin, false); -tk::dnn::Yolo yolo1 (&net, classes, 3, g94_bin); - -tk::dnn::Layer *m95_layers[1] = { &a91 }; -tk::dnn::Route m95 (&net, m95_layers, 1); -tk::dnn::Conv2d c96 (&net, 128, 1, 1, 1, 1, 0, 0, c96_bin, true); -tk::dnn::Activation a96 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Upsample u97 (&net, 2); - -tk::dnn::Layer *m98_layers[2] = { &u97, &s36 }; -tk::dnn::Route m98 (&net, m98_layers, 2); -tk::dnn::Conv2d c99 (&net, 128, 1, 1, 1, 1, 0, 0, c99_bin, true); -tk::dnn::Activation a99 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c100 (&net, 256, 3, 3, 1, 1, 1, 1, c100_bin, true); -tk::dnn::Activation a100 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c101 (&net, 128, 1, 1, 1, 1, 0, 0, c101_bin, true); -tk::dnn::Activation a101 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c102 (&net, 256, 3, 3, 1, 1, 1, 1, c102_bin, true); -tk::dnn::Activation a102 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c103 (&net, 128, 1, 1, 1, 1, 0, 0, c103_bin, true); -tk::dnn::Activation a103 (&net, tk::dnn::ACTIVATION_LEAKY); - -tk::dnn::Conv2d c104 (&net, 256, 3, 3, 1, 1, 1, 1, c104_bin, true); -tk::dnn::Activation a104 (&net, tk::dnn::ACTIVATION_LEAKY); -tk::dnn::Conv2d c105 (&net, preYoloFilters, 1, 1, 1, 1, 0, 0, c105_bin, false); -tk::dnn::Yolo yolo2 (&net, classes, 3, g106_bin); - -yolo[0] = &yolo0; -yolo[1] = &yolo1; -yolo[2] = &yolo2; \ No newline at end of file diff --git a/include/tkDNN/test.h b/include/tkDNN/test.h new file mode 100644 index 0000000..0b810da --- /dev/null +++ b/include/tkDNN/test.h @@ -0,0 +1,68 @@ + +#include +int testInference(std::vector input_bins, std::vector output_bins, + tk::dnn::Network *net, tk::dnn::NetworkRT *netRT = nullptr) { + + std::vector outputs; + for(int i=0; inum_layers; i++) { + if(net->layers[i]->final) + outputs.push_back(net->layers[i]); + } + + // check input + if(input_bins.size() != 1) { + FatalError("currently support only 1 input"); + } + if(output_bins.size() != outputs.size()) { + std::cout<input_dim.tot(), &input_h, &data); + + // outputs + dnnType *cudnn_out[outputs.size()], *rt_out[outputs.size()]; + + tk::dnn::dataDim_t dim1 = net->input_dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); { + dim1.print(); + TIMER_START + net->infer(dim1, data); + TIMER_STOP + dim1.print(); + } + for(int i=0; idstData; + + if(netRT != nullptr) { + tk::dnn::dataDim_t dim2 = net->input_dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); { + dim2.print(); + TIMER_START + netRT->infer(dim2, data); + TIMER_STOP + dim2.print(); + } + for(int i=0; ibuffersRT[i+1]; + } + + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for(int i=0; idstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for (int i = 0; i < 3; i++) - yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for (int j = 0; j < ndets; j++) - { - tk::dnn::Yolo::box b = dets[j].bbox; - int x0 = (b.x - b.w / 2.); - int x1 = (b.x + b.w / 2.); - int y0 = (b.y - b.h / 2.); - int y1 = (b.y + b.h / 2.); - - int cl = 0; - for (int c = 0; c < classes; ++c) - { - float prob = dets[j].prob[c]; - if (prob > 0) - cl = c; - } - std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; - } - TIMER_STOP - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); - { - dim2.print(); - TIMER_START - netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - for (int i = 0; i < 3; i++) - rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; - - int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; - for (int i = 0; i < 3; i++) - { - printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); - dnnType *out, *out_h; - int odim = out_dim[i].tot(); - readBinaryFile(output_bins[i], odim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - } - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/build_models.sh b/tests/build_models.sh deleted file mode 100644 index c6b5fe2..0000000 --- a/tests/build_models.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -if [ "$1" == "download" ]; then - wget https://github.com/ceccocats/tkDNN/releases/download/testData/tkDNN_testwg.tar.gz --no-check-certificate - tar -xf tkDNN_testwg.tar.gz - rm tkDNN_testwg.tar.gz - exit -fi - -echo "build test Model" -cd test -python test_model.py -cd .. -cd mnist -python mnist_model.py -cd .. -echo "export weights" -python weights_exporter.py test/net.h5 --output test/layers -python caffe_weights_exporter.py mnist/lenet.prototxt mnist/lenet.caffemodel --output mnist/layers diff --git a/tests/dla34_cnet/dla34_cnet.cpp b/tests/centernet/dla34_cnet/dla34_cnet.cpp similarity index 100% rename from tests/dla34_cnet/dla34_cnet.cpp rename to tests/centernet/dla34_cnet/dla34_cnet.cpp diff --git a/tests/resnet101_cnet/resnet101_cnet.cpp b/tests/centernet/resnet101_cnet/resnet101_cnet.cpp similarity index 100% rename from tests/resnet101_cnet/resnet101_cnet.cpp rename to tests/centernet/resnet101_cnet/resnet101_cnet.cpp diff --git a/tests/csresnext50-panet-spp/csresnext50-panet-spp.cpp b/tests/csresnext50-panet-spp/csresnext50-panet-spp.cpp deleted file mode 100644 index 11a40ae..0000000 --- a/tests/csresnext50-panet-spp/csresnext50-panet-spp.cpp +++ /dev/null @@ -1,554 +0,0 @@ -#include -#include -#include "tkdnn.h" - -int main() -{ - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 416, 416, 1); - tk::dnn::Network net(dim); - - // create csresnext50-panet-spp model - std::string bin_path = "csresnext50-panet-spp"; - int classes = 80; - tk::dnn::Yolo *yolo[3]; - - std::string input_bin = bin_path + "/layers/input.bin"; - std::string output_bin = bin_path + "/debug/layer137_out.bin"; - std::vector output_bins = { - bin_path + "/debug/layer115_out.bin", - bin_path + "/debug/layer126_out.bin", - bin_path + "/debug/layer137_out.bin"}; - std::string c0_bin = bin_path + "/layers/c0.bin"; - std::string c2_bin = bin_path + "/layers/c2.bin"; - std::string c4_bin = bin_path + "/layers/c4.bin"; - std::string c5_bin = bin_path + "/layers/c5.bin"; - std::string c6_bin = bin_path + "/layers/c6.bin"; - std::string c7_bin = bin_path + "/layers/c7.bin"; - std::string c9_bin = bin_path + "/layers/c9.bin"; - std::string c10_bin = bin_path + "/layers/c10.bin"; - std::string c11_bin = bin_path + "/layers/c11.bin"; - std::string c13_bin = bin_path + "/layers/c13.bin"; - std::string c14_bin = bin_path + "/layers/c14.bin"; - std::string c15_bin = bin_path + "/layers/c15.bin"; - std::string c17_bin = bin_path + "/layers/c17.bin"; - std::string c19_bin = bin_path + "/layers/c19.bin"; - std::string c20_bin = bin_path + "/layers/c20.bin"; - std::string c21_bin = bin_path + "/layers/c21.bin"; - std::string c23_bin = bin_path + "/layers/c23.bin"; - std::string c24_bin = bin_path + "/layers/c24.bin"; - std::string c25_bin = bin_path + "/layers/c25.bin"; - std::string c26_bin = bin_path + "/layers/c26.bin"; - std::string c28_bin = bin_path + "/layers/c28.bin"; - std::string c29_bin = bin_path + "/layers/c29.bin"; - std::string c30_bin = bin_path + "/layers/c30.bin"; - std::string c32_bin = bin_path + "/layers/c32.bin"; - std::string c33_bin = bin_path + "/layers/c33.bin"; - std::string c34_bin = bin_path + "/layers/c34.bin"; - std::string c36_bin = bin_path + "/layers/c36.bin"; - std::string c38_bin = bin_path + "/layers/c38.bin"; - std::string c39_bin = bin_path + "/layers/c39.bin"; - std::string c40_bin = bin_path + "/layers/c40.bin"; - std::string c42_bin = bin_path + "/layers/c42.bin"; - std::string c43_bin = bin_path + "/layers/c43.bin"; - std::string c44_bin = bin_path + "/layers/c44.bin"; - std::string c45_bin = bin_path + "/layers/c45.bin"; - std::string c47_bin = bin_path + "/layers/c47.bin"; - std::string c48_bin = bin_path + "/layers/c48.bin"; - std::string c49_bin = bin_path + "/layers/c49.bin"; - std::string c51_bin = bin_path + "/layers/c51.bin"; - std::string c52_bin = bin_path + "/layers/c52.bin"; - std::string c53_bin = bin_path + "/layers/c53.bin"; - std::string c55_bin = bin_path + "/layers/c55.bin"; - std::string c56_bin = bin_path + "/layers/c56.bin"; - std::string c57_bin = bin_path + "/layers/c57.bin"; - std::string c59_bin = bin_path + "/layers/c59.bin"; - std::string c60_bin = bin_path + "/layers/c60.bin"; - std::string c61_bin = bin_path + "/layers/c61.bin"; - std::string c63_bin = bin_path + "/layers/c63.bin"; - std::string c65_bin = bin_path + "/layers/c65.bin"; - std::string c66_bin = bin_path + "/layers/c66.bin"; - std::string c67_bin = bin_path + "/layers/c67.bin"; - std::string c69_bin = bin_path + "/layers/c69.bin"; - std::string c70_bin = bin_path + "/layers/c70.bin"; - std::string c71_bin = bin_path + "/layers/c71.bin"; - std::string c72_bin = bin_path + "/layers/c72.bin"; - std::string c74_bin = bin_path + "/layers/c74.bin"; - std::string c75_bin = bin_path + "/layers/c75.bin"; - std::string c76_bin = bin_path + "/layers/c76.bin"; - std::string c78_bin = bin_path + "/layers/c78.bin"; - std::string c80_bin = bin_path + "/layers/c80.bin"; - std::string c81_bin = bin_path + "/layers/c81.bin"; - std::string c82_bin = bin_path + "/layers/c82.bin"; - std::string c83_bin = bin_path + "/layers/c83.bin"; - std::string c90_bin = bin_path + "/layers/c90.bin"; - std::string c91_bin = bin_path + "/layers/c91.bin"; - std::string c92_bin = bin_path + "/layers/c92.bin"; - std::string c93_bin = bin_path + "/layers/c93.bin"; - std::string c96_bin = bin_path + "/layers/c96.bin"; - std::string c98_bin = bin_path + "/layers/c98.bin"; - std::string c99_bin = bin_path + "/layers/c99.bin"; - std::string c100_bin = bin_path + "/layers/c100.bin"; - std::string c101_bin = bin_path + "/layers/c101.bin"; - std::string c102_bin = bin_path + "/layers/c102.bin"; - std::string c103_bin = bin_path + "/layers/c103.bin"; - std::string c106_bin = bin_path + "/layers/c106.bin"; - std::string c108_bin = bin_path + "/layers/c108.bin"; - std::string c109_bin = bin_path + "/layers/c109.bin"; - std::string c110_bin = bin_path + "/layers/c110.bin"; - std::string c111_bin = bin_path + "/layers/c111.bin"; - std::string c112_bin = bin_path + "/layers/c112.bin"; - std::string c113_bin = bin_path + "/layers/c113.bin"; - std::string c114_bin = bin_path + "/layers/c114.bin"; - std::string c117_bin = bin_path + "/layers/c117.bin"; - std::string c119_bin = bin_path + "/layers/c119.bin"; - std::string c120_bin = bin_path + "/layers/c120.bin"; - std::string c121_bin = bin_path + "/layers/c121.bin"; - std::string c122_bin = bin_path + "/layers/c122.bin"; - std::string c123_bin = bin_path + "/layers/c123.bin"; - std::string c124_bin = bin_path + "/layers/c124.bin"; - std::string c125_bin = bin_path + "/layers/c125.bin"; - std::string c128_bin = bin_path + "/layers/c128.bin"; - std::string c130_bin = bin_path + "/layers/c130.bin"; - std::string c131_bin = bin_path + "/layers/c131.bin"; - std::string c132_bin = bin_path + "/layers/c132.bin"; - std::string c133_bin = bin_path + "/layers/c133.bin"; - std::string c134_bin = bin_path + "/layers/c134.bin"; - std::string c135_bin = bin_path + "/layers/c135.bin"; - std::string c136_bin = bin_path + "/layers/c136.bin"; - std::string g115_bin = bin_path + "/layers/g115.bin"; - std::string g126_bin = bin_path + "/layers/g126.bin"; - std::string g137_bin = bin_path + "/layers/g137.bin"; - - downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download"); - - tk::dnn::Conv2d c0(&net, 64, 7, 7, 2, 2, 3, 3, c0_bin, true); - tk::dnn::Activation a0(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2(&net, 128, 1, 1, 1, 1, 0, 0, c2_bin, true); - tk::dnn::Activation a2(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *r3_layers[1] = {&p1}; - tk::dnn::Route r3(&net, r3_layers, 1); - - tk::dnn::Conv2d c4(&net, 64, 1, 1, 1, 1, 0, 0, c4_bin, true); - tk::dnn::Activation a4(&net, tk::dnn::ACTIVATION_LEAKY); - - // //1-1 - tk::dnn::Conv2d c5(&net, 128, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c6(&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true, false, 32, false); - tk::dnn::Activation a6(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c7(&net, 128, 1, 1, 1, 1, 0, 0, c7_bin, true); - - tk::dnn::Shortcut s8(&net, &a4); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_LEAKY); - - //1-2 - tk::dnn::Conv2d c9(&net, 128, 1, 1, 1, 1, 0, 0, c9_bin, true); - tk::dnn::Activation a9(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c10(&net, 128, 3, 3, 1, 1, 1, 1, c10_bin, true, false, 32); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c11(&net, 128, 1, 1, 1, 1, 0, 0, c11_bin, true); - - tk::dnn::Shortcut s12(&net, &a8); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - - //1-3 - tk::dnn::Conv2d c13(&net, 128, 1, 1, 1, 1, 0, 0, c13_bin, true); - tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c14(&net, 128, 3, 3, 1, 1, 1, 1, c14_bin, true, false, 32); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c15(&net, 128, 1, 1, 1, 1, 0, 0, c15_bin, true); - - tk::dnn::Shortcut s16(&net, &a12); - tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_LEAKY); - - // //1-T - tk::dnn::Conv2d c17(&net, 128, 1, 1, 1, 1, 0, 0, c17_bin, true); - tk::dnn::Activation a17(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *r18_layers[2] = {&a17, &a2}; - tk::dnn::Route r18(&net, r18_layers, 2); - - tk::dnn::Conv2d c19(&net, 256, 1, 1, 1, 1, 0, 0, c19_bin, true); - tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c20(&net, 256, 3, 3, 2, 2, 1, 1, c20_bin, true, false, 32); - tk::dnn::Activation a20(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c21(&net, 256, 1, 1, 1, 1, 0, 0, c21_bin, true); - - tk::dnn::Layer *r22_layers[2] = {&a20}; - tk::dnn::Route r22(&net, r22_layers, 1); - - tk::dnn::Conv2d c23(&net, 256, 1, 1, 1, 1, 0, 0, c23_bin, true); - - //2-1 - tk::dnn::Conv2d c24(&net, 256, 1, 1, 1, 1, 0, 0, c24_bin, true); - tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c25(&net, 256, 3, 3, 1, 1, 1, 1, c25_bin, true, false, 32); - tk::dnn::Activation a25(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c26(&net, 256, 1, 1, 1, 1, 0, 0, c26_bin, true); - - tk::dnn::Shortcut s27(&net, &c23); - tk::dnn::Activation a27(&net, tk::dnn::ACTIVATION_LEAKY); - - //2-2 - tk::dnn::Conv2d c28(&net, 256, 1, 1, 1, 1, 0, 0, c28_bin, true); - tk::dnn::Activation a28(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c29(&net, 256, 3, 3, 1, 1, 1, 1, c29_bin, true, false, 32); - tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c30(&net, 256, 1, 1, 1, 1, 0, 0, c30_bin, true); - - tk::dnn::Shortcut s31(&net, &a27); - tk::dnn::Activation a31(&net, tk::dnn::ACTIVATION_LEAKY); - - //2-3 - tk::dnn::Conv2d c32(&net, 256, 1, 1, 1, 1, 0, 0, c32_bin, true); - tk::dnn::Activation a32(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c33(&net, 256, 3, 3, 1, 1, 1, 1, c33_bin, true, false, 32); - tk::dnn::Activation a33(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c34(&net, 256, 1, 1, 1, 1, 0, 0, c34_bin, true); - - tk::dnn::Shortcut s35(&net, &a31); - tk::dnn::Activation a35(&net, tk::dnn::ACTIVATION_LEAKY); - - // //2-T - tk::dnn::Conv2d c36(&net, 256, 1, 1, 1, 1, 0, 0, c36_bin, true); - tk::dnn::Activation a36(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *r37_layers[2] = {&a36, &c21}; - tk::dnn::Route r37(&net, r37_layers, 2); - - tk::dnn::Conv2d c38(&net, 512, 1, 1, 1, 1, 0, 0, c38_bin, true); - tk::dnn::Activation a38(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c39(&net, 512, 3, 3, 2, 2, 1, 1, c39_bin, true, false, 32); - tk::dnn::Activation a39(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c40(&net, 512, 1, 1, 1, 1, 0, 0, c40_bin, true); - - tk::dnn::Layer *r41_layers[2] = {&a39}; - tk::dnn::Route r41(&net, r41_layers, 1); - - tk::dnn::Conv2d c42(&net, 512, 1, 1, 1, 1, 0, 0, c42_bin, true); - - //3-1 - tk::dnn::Conv2d c43(&net, 512, 1, 1, 1, 1, 0, 0, c43_bin, true); - tk::dnn::Activation a43(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c44(&net, 512, 3, 3, 1, 1, 1, 1, c44_bin, true, false, 32); - tk::dnn::Activation a44(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c45(&net, 512, 1, 1, 1, 1, 0, 0, c45_bin, true); - - tk::dnn::Shortcut s46(&net, &c42); - tk::dnn::Activation a46(&net, tk::dnn::ACTIVATION_LEAKY); - - //3-2 - tk::dnn::Conv2d c47(&net, 512, 1, 1, 1, 1, 0, 0, c47_bin, true); - tk::dnn::Activation a47(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c48(&net, 512, 3, 3, 1, 1, 1, 1, c48_bin, true, false, 32); - tk::dnn::Activation a48(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c49(&net, 512, 1, 1, 1, 1, 0, 0, c49_bin, true); - - tk::dnn::Shortcut s50(&net, &a46); - tk::dnn::Activation a50(&net, tk::dnn::ACTIVATION_LEAKY); - - //3-3 - tk::dnn::Conv2d c51(&net, 512, 1, 1, 1, 1, 0, 0, c51_bin, true); - tk::dnn::Activation a51(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c52(&net, 512, 3, 3, 1, 1, 1, 1, c52_bin, true, false, 32); - tk::dnn::Activation a52(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c53(&net, 512, 1, 1, 1, 1, 0, 0, c53_bin, true); - - tk::dnn::Shortcut s54(&net, &a50); - tk::dnn::Activation a54(&net, tk::dnn::ACTIVATION_LEAKY); - - //3-4 - tk::dnn::Conv2d c55(&net, 512, 1, 1, 1, 1, 0, 0, c55_bin, true); - tk::dnn::Activation a55(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c56(&net, 512, 3, 3, 1, 1, 1, 1, c56_bin, true, false, 32); - tk::dnn::Activation a56(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c57(&net, 512, 1, 1, 1, 1, 0, 0, c57_bin, true); - - tk::dnn::Shortcut s58(&net, &a54); - tk::dnn::Activation a58(&net, tk::dnn::ACTIVATION_LEAKY); - - //3-5 - tk::dnn::Conv2d c59(&net, 512, 1, 1, 1, 1, 0, 0, c59_bin, true); - tk::dnn::Activation a59(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c60(&net, 512, 3, 3, 1, 1, 1, 1, c60_bin, true, false, 32); - tk::dnn::Activation a60(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c61(&net, 512, 1, 1, 1, 1, 0, 0, c61_bin, true); - - tk::dnn::Shortcut s62(&net, &a58); - tk::dnn::Activation a62(&net, tk::dnn::ACTIVATION_LEAKY); - - //3-T - tk::dnn::Conv2d c63(&net, 512, 1, 1, 1, 1, 0, 0, c63_bin, true); - tk::dnn::Activation a63(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *r64_layers[2] = {&a63, &c40}; - tk::dnn::Route r64(&net, r64_layers, 2); - - tk::dnn::Conv2d c65(&net, 1024, 1, 1, 1, 1, 0, 0, c65_bin, true); - tk::dnn::Activation a65(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c66(&net, 1024, 3, 3, 2, 2, 1, 1, c66_bin, true, false, 32); - tk::dnn::Activation a66(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c67(&net, 1024, 1, 1, 1, 1, 0, 0, c67_bin, true); - tk::dnn::Activation a67(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *r68_layers[2] = {&a66}; - tk::dnn::Route r68(&net, r68_layers, 1); - - tk::dnn::Conv2d c69(&net, 1024, 1, 1, 1, 1, 0, 0, c69_bin, true); - tk::dnn::Activation a69(&net, tk::dnn::ACTIVATION_LEAKY); - - //4-1 - tk::dnn::Conv2d c70(&net, 1024, 1, 1, 1, 1, 0, 0, c70_bin, true); - tk::dnn::Activation a70(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c71(&net, 1024, 3, 3, 1, 1, 1, 1, c71_bin, true, false, 32); - tk::dnn::Activation a71(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c72(&net, 1024, 1, 1, 1, 1, 0, 0, c72_bin, true); - - tk::dnn::Shortcut s73(&net, &a69); - tk::dnn::Activation a73(&net, tk::dnn::ACTIVATION_LEAKY); - - //4-2 - tk::dnn::Conv2d c74(&net, 1024, 1, 1, 1, 1, 0, 0, c74_bin, true); - tk::dnn::Activation a74(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c75(&net, 1024, 3, 3, 1, 1, 1, 1, c75_bin, true, false, 32); - tk::dnn::Activation a75(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c76(&net, 1024, 1, 1, 1, 1, 0, 0, c76_bin, true); - - tk::dnn::Shortcut s77(&net, &a73); - tk::dnn::Activation a77(&net, tk::dnn::ACTIVATION_LEAKY); - - //4-T - tk::dnn::Conv2d c78(&net, 1024, 1, 1, 1, 1, 0, 0, c78_bin, true); - tk::dnn::Activation a78(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *r79_layers[2] = {&a78, &a67}; - tk::dnn::Route r79(&net, r79_layers, 2); - - tk::dnn::Conv2d c80(&net, 2048, 1, 1, 1, 1, 0, 0, c80_bin, true); - tk::dnn::Activation a80(&net, tk::dnn::ACTIVATION_LEAKY); - - // //////////////////// - - tk::dnn::Conv2d c81(&net, 512, 1, 1, 1, 1, 0, 0, c81_bin, true); - tk::dnn::Activation a81(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c82(&net, 1024, 3, 3, 1, 1, 1, 1, c82_bin, true); - tk::dnn::Activation a82(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c83(&net, 512, 1, 1, 1, 1, 0, 0, c83_bin, true); - tk::dnn::Activation a83(&net, tk::dnn::ACTIVATION_LEAKY); - - //SPP - tk::dnn::Pooling p84(&net, 5, 5, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r85_layers[1] = {&a83}; - tk::dnn::Route r85(&net, r85_layers, 1); - - tk::dnn::Pooling p86(&net, 9, 9, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r87_layers[1] = {&a83}; - tk::dnn::Route r87(&net, r87_layers, 1); - - tk::dnn::Pooling p88(&net, 13, 13, 1, 1, 12, 12, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r89_layers[4] = {&p88, &p86, &p84, &a83}; - tk::dnn::Route r89(&net, r89_layers, 4); - //END SPP - - tk::dnn::Conv2d c90(&net, 512, 1, 1, 1, 1, 0, 0, c90_bin, true); - tk::dnn::Activation a90(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c91(&net, 1024, 3, 3, 1, 1, 1, 1, c91_bin, true); - tk::dnn::Activation a91(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c92(&net, 512, 1, 1, 1, 1, 0, 0, c92_bin, true); - tk::dnn::Activation a92(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c93(&net, 256, 1, 1, 1, 1, 0, 0, c93_bin, true); - tk::dnn::Activation a93(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u94(&net, 2); - tk::dnn::Layer *r95_layers[1] = {&a65}; - tk::dnn::Route r95(&net, r95_layers, 1); - tk::dnn::Conv2d c96(&net, 256, 1, 1, 1, 1, 0, 0, c96_bin, true); - tk::dnn::Activation a96(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r97_layers[2] = {&a96,&u94}; - tk::dnn::Route r97(&net, r97_layers, 2); - - tk::dnn::Conv2d c98(&net, 256, 1, 1, 1, 1, 0, 0, c98_bin, true); - tk::dnn::Activation a98(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c99(&net, 512, 3, 3, 1, 1, 1, 1, c99_bin, true); - tk::dnn::Activation a99(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c100(&net, 256, 1, 1, 1, 1, 0, 0, c100_bin, true); - tk::dnn::Activation a100(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c101(&net, 512, 3, 3, 1, 1, 1, 1, c101_bin, true); - tk::dnn::Activation a101(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c102(&net, 256, 1, 1, 1, 1, 0, 0, c102_bin, true); - tk::dnn::Activation a102(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c103(&net, 128, 1, 1, 1, 1, 0, 0, c103_bin, true); - tk::dnn::Activation a103(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u104(&net, 2); - tk::dnn::Layer *r105_layers[1] = {&a38}; - tk::dnn::Route r105(&net, r105_layers, 1); - tk::dnn::Conv2d c106(&net, 128, 1, 1, 1, 1, 0, 0, c106_bin, true); - tk::dnn::Activation a106(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r107_layers[2] = {&a106,&u104}; - tk::dnn::Route r107(&net, r107_layers, 2); - - - tk::dnn::Conv2d c108(&net, 128, 1, 1, 1, 1, 0, 0, c108_bin, true); - tk::dnn::Activation a108(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c109(&net, 256, 3, 3, 1, 1, 1, 1, c109_bin, true); - tk::dnn::Activation a109(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c110(&net, 128, 1, 1, 1, 1, 0, 0, c110_bin, true); - tk::dnn::Activation a110(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c111(&net, 256, 3, 3, 1, 1, 1, 1, c111_bin, true); - tk::dnn::Activation a111(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c112(&net, 128, 1, 1, 1, 1, 0, 0, c112_bin, true); - tk::dnn::Activation a112(&net, tk::dnn::ACTIVATION_LEAKY); - - // ########################### - - tk::dnn::Conv2d c113(&net, 256, 3, 3, 1, 1, 1, 1, c113_bin, true); - tk::dnn::Activation a113(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c114(&net, 255, 1, 1, 1, 1, 0, 0, c114_bin, false); - tk::dnn::Yolo yolo115(&net, classes, 3, g115_bin); - - tk::dnn::Layer *r116_layers[1] = {&a112}; - tk::dnn::Route r116(&net, r116_layers, 1); - tk::dnn::Conv2d c117(&net, 256, 3, 3, 2, 2, 1, 1, c117_bin, true); - tk::dnn::Activation a117(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r118_layers[2] = {&a117,&a102}; - tk::dnn::Route r118(&net, r118_layers, 2); - - tk::dnn::Conv2d c119(&net, 256, 1, 1, 1, 1, 0, 0, c119_bin, true); - tk::dnn::Activation a119(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c120(&net, 512, 3, 3, 1, 1, 1, 1, c120_bin, true); - tk::dnn::Activation a120(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c121(&net, 256, 1, 1, 1, 1, 0, 0, c121_bin, true); - tk::dnn::Activation a121(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c122(&net, 512, 3, 3, 1, 1, 1, 1, c122_bin, true); - tk::dnn::Activation a122(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c123(&net, 256, 1, 1, 1, 1, 0, 0, c123_bin, true); - tk::dnn::Activation a123(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c124(&net, 512, 3, 3, 1, 1, 1, 1, c124_bin, true); - tk::dnn::Activation a124(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c125(&net, 255, 1, 1, 1, 1, 0, 0, c125_bin, false); - tk::dnn::Yolo yolo126(&net, classes, 3, g126_bin); - - tk::dnn::Layer *r127_layers[1] = {&a123}; - tk::dnn::Route r127(&net, r127_layers, 1); - tk::dnn::Conv2d c128(&net, 512, 3, 3, 2, 2, 1, 1, c128_bin, true); - tk::dnn::Activation a128(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r129_layers[2] = {&a128,&a92}; - tk::dnn::Route r129(&net, r129_layers, 2); - - tk::dnn::Conv2d c130(&net, 512, 1, 1, 1, 1, 0, 0, c130_bin, true); - tk::dnn::Activation a130(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c131(&net, 1024, 3, 3, 1, 1, 1, 1, c131_bin, true); - tk::dnn::Activation a131(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c132(&net, 512, 1, 1, 1, 1, 0, 0, c132_bin, true); - tk::dnn::Activation a132(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c133(&net, 1024, 3, 3, 1, 1, 1, 1, c133_bin, true); - tk::dnn::Activation a133(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c134(&net, 512, 1, 1, 1, 1, 0, 0, c134_bin, true); - tk::dnn::Activation a134(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c135(&net, 1024, 3, 3, 1, 1, 1, 1, c135_bin, true); - tk::dnn::Activation a135(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c136(&net, 255, 1, 1, 1, 1, 0, 0, c136_bin, false); - tk::dnn::Yolo yolo137(&net, classes, 3, g137_bin); - - yolo[0] = &yolo115; - yolo[1] = &yolo126; - yolo[2] = &yolo137; - - // fill classes names - for (int i = 0; i < 3; i++) - { - yolo[i]->classesNames = {"person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"}; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - // //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("csresnext50-panet-spp")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for (int i = 0; i < 3; i++) - out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); - { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - for (int i = 0; i < 3; i++) - cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for (int i = 0; i < 3; i++) - yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for (int j = 0; j < ndets; j++) - { - tk::dnn::Yolo::box b = dets[j].bbox; - int x0 = (b.x - b.w / 2.); - int x1 = (b.x + b.w / 2.); - int y0 = (b.y - b.h / 2.); - int y1 = (b.y + b.h / 2.); - - int cl = 0; - for (int c = 0; c < classes; ++c) - { - float prob = dets[j].prob[c]; - if (prob > 0) - cl = c; - } - std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; - } - TIMER_STOP - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); - { - dim2.print(); - TIMER_START - netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - for (int i = 0; i < 3; i++) - rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; - - int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; - for (int i = 0; i < 3; i++) - { - printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); - dnnType *out, *out_h; - int odim = out_dim[i].tot(); - readBinaryFile(output_bins[i], odim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - } - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/csresnext50-panet-spp/csresnext50-panet-spp.cfg b/tests/darknet/cfg/csresnext50-panet-spp.cfg similarity index 100% rename from tests/csresnext50-panet-spp/csresnext50-panet-spp.cfg rename to tests/darknet/cfg/csresnext50-panet-spp.cfg diff --git a/tests/bdd-csresnext50-panet-spp/berkeleycsresnetx50.cfg b/tests/darknet/cfg/csresnext50-panet-spp_berkeley.cfg similarity index 100% rename from tests/bdd-csresnext50-panet-spp/berkeleycsresnetx50.cfg rename to tests/darknet/cfg/csresnext50-panet-spp_berkeley.cfg diff --git a/tests/yolo/yolo.cfg b/tests/darknet/cfg/yolo2.cfg similarity index 100% rename from tests/yolo/yolo.cfg rename to tests/darknet/cfg/yolo2.cfg diff --git a/tests/yolo_voc/yolo_voc.cfg b/tests/darknet/cfg/yolo2_voc.cfg similarity index 100% rename from tests/yolo_voc/yolo_voc.cfg rename to tests/darknet/cfg/yolo2_voc.cfg diff --git a/tests/yolo_tiny/tiny-yolo.cfg b/tests/darknet/cfg/yolo2tiny.cfg similarity index 100% rename from tests/yolo_tiny/tiny-yolo.cfg rename to tests/darknet/cfg/yolo2tiny.cfg diff --git a/tests/yolo3/yolov3.cfg b/tests/darknet/cfg/yolo3.cfg similarity index 100% rename from tests/yolo3/yolov3.cfg rename to tests/darknet/cfg/yolo3.cfg diff --git a/tests/yolo3_512tp/yolo3512.cfg b/tests/darknet/cfg/yolo3_512.cfg similarity index 93% rename from tests/yolo3_512tp/yolo3512.cfg rename to tests/darknet/cfg/yolo3_512.cfg index 00ea6e9..032d49a 100644 --- a/tests/yolo3_512tp/yolo3512.cfg +++ b/tests/darknet/cfg/yolo3_512.cfg @@ -1,10 +1,10 @@ [net] # Testing -#batch=1 -#subdivisions=1 +# batch=1 +# subdivisions=1 # Training -batch=16 -subdivisions=1 +batch=32 +subdivisions=32 width=512 height=512 channels=3 @@ -600,14 +600,14 @@ activation=leaky size=1 stride=1 pad=1 -filters=24 +filters=255 activation=linear [yolo] mask = 6,7,8 -anchors = 10.256,16.494, 11.724,18.558, 17.678,16.437, 25.619,14.985, 46.845,79.02, 58.643,81.204, 23.646,208.56, 30.837,211.57, 37.921,211.16 -classes=3 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 num=9 jitter=.3 ignore_thresh = .7 @@ -633,6 +633,7 @@ stride=2 layers = -1, 61 + [convolutional] batch_normalize=1 filters=256 @@ -685,14 +686,14 @@ activation=leaky size=1 stride=1 pad=1 -filters=24 +filters=255 activation=linear [yolo] mask = 3,4,5 -anchors = 10.256,16.494, 11.724,18.558, 17.678,16.437, 25.619,14.985, 46.845,79.02, 58.643,81.204, 23.646,208.56, 30.837,211.57, 37.921,211.16 -classes=3 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 num=9 jitter=.3 ignore_thresh = .7 @@ -772,16 +773,17 @@ activation=leaky size=1 stride=1 pad=1 -filters=24 +filters=255 activation=linear [yolo] mask = 0,1,2 -anchors = 10.256,16.494, 11.724,18.558, 17.678,16.437, 25.619,14.985, 46.845,79.02, 58.643,81.204, 23.646,208.56, 30.837,211.57, 37.921,211.16 -classes=3 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 num=9 jitter=.3 ignore_thresh = .7 truth_thresh = 1 random=1 + diff --git a/tests/yolo3_berkeley/yolo3_berkeley.cfg b/tests/darknet/cfg/yolo3_berkeley.cfg similarity index 100% rename from tests/yolo3_berkeley/yolo3_berkeley.cfg rename to tests/darknet/cfg/yolo3_berkeley.cfg diff --git a/tests/yolo3_coco4/yolov3-coco4.cfg b/tests/darknet/cfg/yolo3_coco4.cfg similarity index 100% rename from tests/yolo3_coco4/yolov3-coco4.cfg rename to tests/darknet/cfg/yolo3_coco4.cfg diff --git a/tests/yolo3_flir/yolo3_flir.cfg b/tests/darknet/cfg/yolo3_flir.cfg similarity index 100% rename from tests/yolo3_flir/yolo3_flir.cfg rename to tests/darknet/cfg/yolo3_flir.cfg diff --git a/tests/yolo3_tiny/yolov3-tiny.cfg b/tests/darknet/cfg/yolo3tiny.cfg similarity index 100% rename from tests/yolo3_tiny/yolov3-tiny.cfg rename to tests/darknet/cfg/yolo3tiny.cfg diff --git a/tests/yolo3_tiny512tp/yolo3tiny512.cfg b/tests/darknet/cfg/yolo3tiny_512.cfg similarity index 87% rename from tests/yolo3_tiny512tp/yolo3tiny512.cfg rename to tests/darknet/cfg/yolo3tiny_512.cfg index baecead..049a3a6 100644 --- a/tests/yolo3_tiny512tp/yolo3tiny512.cfg +++ b/tests/darknet/cfg/yolo3tiny_512.cfg @@ -124,15 +124,15 @@ activation=leaky size=1 stride=1 pad=1 -filters=24 +filters=255 activation=linear [yolo] mask = 3,4,5 -anchors = 10.638,16.801, 13.183,19.091, 24.568,12.24, 54.462,77.421, 29.199,210.49, 37.495,212.21 -classes=3 +anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319 +classes=80 num=6 jitter=.3 ignore_thresh = .7 @@ -168,13 +168,13 @@ activation=leaky size=1 stride=1 pad=1 -filters=24 +filters=255 activation=linear [yolo] mask = 0,1,2 -anchors = 10.638,16.801, 13.183,19.091, 24.568,12.24, 54.462,77.421, 29.199,210.49, 37.495,212.21 -classes=3 +anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319 +classes=80 num=6 jitter=.3 ignore_thresh = .7 diff --git a/tests/yolo4/yolov4.cfg b/tests/darknet/cfg/yolo4.cfg similarity index 100% rename from tests/yolo4/yolov4.cfg rename to tests/darknet/cfg/yolo4.cfg diff --git a/tests/darknet/csresnext50-panet-spp.cpp b/tests/darknet/csresnext50-panet-spp.cpp new file mode 100644 index 0000000..f2f66f6 --- /dev/null +++ b/tests/darknet/csresnext50-panet-spp.cpp @@ -0,0 +1,33 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "csresnext50-panet-spp"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer115_out.bin", + bin_path + "/debug/layer126_out.bin", + bin_path + "/debug/layer137_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/csresnext50-panet-spp.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/csresnext50-panet-spp_berkeley.cpp b/tests/darknet/csresnext50-panet-spp_berkeley.cpp new file mode 100644 index 0000000..ca21399 --- /dev/null +++ b/tests/darknet/csresnext50-panet-spp_berkeley.cpp @@ -0,0 +1,33 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "bdd-csresnext50-panet-spp"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer115_out.bin", + bin_path + "/debug/layer126_out.bin", + bin_path + "/debug/layer137_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/csresnext50-panet-spp_berkeley.cfg"; + std::string name_path = "../tests/darknet/names/berkeley.names"; + // downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s//download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/yolo3/berkeley.names b/tests/darknet/names/berkeley.names similarity index 100% rename from tests/yolo3/berkeley.names rename to tests/darknet/names/berkeley.names diff --git a/tests/yolo3/coco.names b/tests/darknet/names/coco.names similarity index 100% rename from tests/yolo3/coco.names rename to tests/darknet/names/coco.names diff --git a/tests/darknet/names/coco4.names b/tests/darknet/names/coco4.names new file mode 100644 index 0000000..82cb5c4 --- /dev/null +++ b/tests/darknet/names/coco4.names @@ -0,0 +1,4 @@ +person +bicycle +car +motorbike diff --git a/tests/darknet/names/flir.names b/tests/darknet/names/flir.names new file mode 100644 index 0000000..03f4d8a --- /dev/null +++ b/tests/darknet/names/flir.names @@ -0,0 +1,3 @@ +person +bike +car diff --git a/tests/yolo3/voc.names b/tests/darknet/names/voc.names similarity index 100% rename from tests/yolo3/voc.names rename to tests/darknet/names/voc.names diff --git a/tests/darknet/yolo2.cpp b/tests/darknet/yolo2.cpp new file mode 100644 index 0000000..e6b40a5 --- /dev/null +++ b/tests/darknet/yolo2.cpp @@ -0,0 +1,31 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo2"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "layers/output.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo2.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/nf4PJ3k8bxBETwL/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo2_voc.cpp b/tests/darknet/yolo2_voc.cpp new file mode 100644 index 0000000..2efcbcb --- /dev/null +++ b/tests/darknet/yolo2_voc.cpp @@ -0,0 +1,32 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo2_voc"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/layers/output.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo2_voc.cfg"; + std::string name_path = "../tests/darknet/names/voc.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/DJC5Fi2pEjfNDP9/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} + diff --git a/tests/darknet/yolo2tiny.cpp b/tests/darknet/yolo2tiny.cpp new file mode 100644 index 0000000..fefdaa5 --- /dev/null +++ b/tests/darknet/yolo2tiny.cpp @@ -0,0 +1,31 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo2tiny"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "layers/output.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo2tiny.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/nf4PJ3k8bxBETwL/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo3.cpp b/tests/darknet/yolo3.cpp new file mode 100644 index 0000000..b96d79a --- /dev/null +++ b/tests/darknet/yolo3.cpp @@ -0,0 +1,33 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo3"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer82_out.bin", + bin_path + "/debug/layer94_out.bin", + bin_path + "/debug/layer106_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo3.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo3_512.cpp b/tests/darknet/yolo3_512.cpp new file mode 100644 index 0000000..11a7839 --- /dev/null +++ b/tests/darknet/yolo3_512.cpp @@ -0,0 +1,33 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo3_512"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer82_out.bin", + bin_path + "/debug/layer94_out.bin", + bin_path + "/debug/layer106_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo3_512.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/RGecMeGLD4cXEWL/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo3_berkeley.cpp b/tests/darknet/yolo3_berkeley.cpp new file mode 100644 index 0000000..6c3512c --- /dev/null +++ b/tests/darknet/yolo3_berkeley.cpp @@ -0,0 +1,33 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo3_berkeley"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer82_out.bin", + bin_path + "/debug/layer94_out.bin", + bin_path + "/debug/layer106_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo3_berkeley.cfg"; + std::string name_path = "../tests/darknet/names/barkeley.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/o5cHa4AjTKS64oD/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo3_coco4.cpp b/tests/darknet/yolo3_coco4.cpp new file mode 100644 index 0000000..cf69a26 --- /dev/null +++ b/tests/darknet/yolo3_coco4.cpp @@ -0,0 +1,33 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo3_coco4"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer82_out.bin", + bin_path + "/debug/layer94_out.bin", + bin_path + "/debug/layer106_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo3_coco4.cfg"; + std::string name_path = "../tests/darknet/names/coco4.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/o27NDzSAartbyc4/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo3_flir.cpp b/tests/darknet/yolo3_flir.cpp new file mode 100644 index 0000000..fd5c1d5 --- /dev/null +++ b/tests/darknet/yolo3_flir.cpp @@ -0,0 +1,33 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo3_flir"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer82_out.bin", + bin_path + "/debug/layer94_out.bin", + bin_path + "/debug/layer106_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo3_flir.cfg"; + std::string name_path = "../tests/darknet/names/flir.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/62DECncmF6bMMiH/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo3tiny.cpp b/tests/darknet/yolo3tiny.cpp new file mode 100644 index 0000000..247b152 --- /dev/null +++ b/tests/darknet/yolo3tiny.cpp @@ -0,0 +1,31 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo3tiny"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "debug/layer23_out.bin", + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo3tiny.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/LMcSHtWaLeps8yN/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo3tiny512.cpp b/tests/darknet/yolo3tiny512.cpp new file mode 100644 index 0000000..495033a --- /dev/null +++ b/tests/darknet/yolo3tiny512.cpp @@ -0,0 +1,31 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo3tiny_512"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "debug/layer23_out.bin", + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo3tiny_512.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/8Zt6bHwHADqP4JC/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo4.cpp b/tests/darknet/yolo4.cpp new file mode 100644 index 0000000..70257d2 --- /dev/null +++ b/tests/darknet/yolo4.cpp @@ -0,0 +1,33 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo4.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + delete net; + delete netRT; + return ret; +} diff --git a/tests/caffe_weights_exporter.py b/tests/exporters/caffe_weights_exporter.py similarity index 100% rename from tests/caffe_weights_exporter.py rename to tests/exporters/caffe_weights_exporter.py diff --git a/tests/weights_exporter.py b/tests/exporters/keras_weights_exporter.py similarity index 100% rename from tests/weights_exporter.py rename to tests/exporters/keras_weights_exporter.py diff --git a/tests/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp b/tests/mobilenet/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp similarity index 100% rename from tests/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp rename to tests/mobilenet/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp diff --git a/tests/mobilenetv2ssd/mobilenetv2ssd.cpp b/tests/mobilenet/mobilenetv2ssd/mobilenetv2ssd.cpp similarity index 100% rename from tests/mobilenetv2ssd/mobilenetv2ssd.cpp rename to tests/mobilenet/mobilenetv2ssd/mobilenetv2ssd.cpp diff --git a/tests/mobilenetv2ssd512/mobilenetv2ssd512.cpp b/tests/mobilenet/mobilenetv2ssd512/mobilenetv2ssd512.cpp similarity index 100% rename from tests/mobilenetv2ssd512/mobilenetv2ssd512.cpp rename to tests/mobilenet/mobilenetv2ssd512/mobilenetv2ssd512.cpp diff --git a/tests/yolo/yolo.cpp b/tests/yolo/yolo.cpp deleted file mode 100644 index e47fd8f..0000000 --- a/tests/yolo/yolo.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include -#include "tkdnn.h" - -const char *input_bin = "yolo/layers/input.bin"; -const char *c0_bin = "yolo/layers/c0.bin"; -const char *c2_bin = "yolo/layers/c2.bin"; -const char *c4_bin = "yolo/layers/c4.bin"; -const char *c5_bin = "yolo/layers/c5.bin"; -const char *c6_bin = "yolo/layers/c6.bin"; -const char *c8_bin = "yolo/layers/c8.bin"; -const char *c9_bin = "yolo/layers/c9.bin"; -const char *c10_bin = "yolo/layers/c10.bin"; -const char *c12_bin = "yolo/layers/c12.bin"; -const char *c13_bin = "yolo/layers/c13.bin"; -const char *c14_bin = "yolo/layers/c14.bin"; -const char *c15_bin = "yolo/layers/c15.bin"; -const char *c16_bin = "yolo/layers/c16.bin"; -const char *c18_bin = "yolo/layers/c18.bin"; -const char *c19_bin = "yolo/layers/c19.bin"; -const char *c20_bin = "yolo/layers/c20.bin"; -const char *c21_bin = "yolo/layers/c21.bin"; -const char *c22_bin = "yolo/layers/c22.bin"; -const char *c23_bin = "yolo/layers/c23.bin"; -const char *c24_bin = "yolo/layers/c24.bin"; -const char *c26_bin = "yolo/layers/c26.bin"; -const char *c29_bin = "yolo/layers/c29.bin"; -const char *c30_bin = "yolo/layers/c30.bin"; -const char *g31_bin = "yolo/layers/g31.bin"; -const char *output_bin = "yolo/layers/output.bin"; - -int main() { - - downloadWeightsifDoNotExist(input_bin, "yolo", "https://cloud.hipert.unimore.it/s/nf4PJ3k8bxBETwL/download"); - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 608, 608, 1); - tk::dnn::Network net(dim); - - tk::dnn::Conv2d c0 (&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 64, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 128, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c5 (&net, 64, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p7 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8 (&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c9 (&net, 128, 1, 1, 1, 1, 0, 0, c9_bin, true); - tk::dnn::Activation a9 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c10(&net, 256, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p11(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c12(&net, 512, 3, 3, 1, 1, 1, 1, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c13(&net, 256, 1, 1, 1, 1, 0, 0, c13_bin, true); - tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c14(&net, 512, 3, 3, 1, 1, 1, 1, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c15(&net, 256, 1, 1, 1, 1, 0, 0, c15_bin, true); - tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c16(&net, 512, 3, 3, 1, 1, 1, 1, c16_bin, true); - tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p17(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c19(&net, 512, 1, 1, 1, 1, 0, 0, c19_bin, true); - tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true); - tk::dnn::Activation a20(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c21(&net, 512, 1, 1, 1, 1, 0, 0, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true); - tk::dnn::Activation a22(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true); - tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true); - tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *m25_layers[1] = { &a16 }; - tk::dnn::Route m25(&net, m25_layers, 1); - tk::dnn::Conv2d c26(&net, 64, 1, 1, 1, 1, 0, 0, c26_bin, true); - tk::dnn::Activation a26(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Reorg r27(&net, 2); - - tk::dnn::Layer *m28_layers[2] = { &r27, &a24 }; - tk::dnn::Route m28(&net, m28_layers, 2); - - tk::dnn::Conv2d c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true); - tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c30(&net, 425, 1, 1, 1, 1, 0, 0, c30_bin, false); - tk::dnn::Region g31(&net, 80, 4, 5); - - tk::dnn::RegionInterpret rI(dim, g31.output_dim, 80, 4, 5, 0.6f, g31_bin); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - - // std::cout<<"\n\nDetected objects: \n"; - // dnnType *output_h = new dnnType[rI.output_dim.tot()]; - // checkCuda(cudaMemcpy(output_h, out_data2, - // rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost)); - // rI.interpretData(output_h); - // rI.showImageResult(input_h); - - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo3/yolo3.cpp b/tests/yolo3/yolo3.cpp deleted file mode 100644 index 89b8283..0000000 --- a/tests/yolo3/yolo3.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#include -#include -#include "tkdnn.h" -#include "DarknetParser.h" - -int main() { - - // create yolo3 model - std::string bin_path = "yolo3"; - downloadWeightsifDoNotExist("yolo3/layers/input.bin", bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); - - tk::dnn::Network *net = tk::dnn::darknetParser("../tests/yolo3/yolov3.cfg", "yolo3/layers", "../tests/yolo3/coco.names"); - net->print(); - - std::vector yolo; - for(int i=0; inum_layers; i++) { - if(net->layers[i]->getLayerType() == tk::dnn::layerType_t::LAYER_YOLO) - yolo.push_back((tk::dnn::Yolo*)net->layers[i]); - } - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(net, net->getNetworkRTName("yolo3")); - - - std::string input_bin = bin_path + "/layers/input.bin"; - std::vector output_bins = { - bin_path + "/debug/layer82_out.bin", - bin_path + "/debug/layer94_out.bin", - bin_path + "/debug/layer106_out.bin" - }; - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, net->input_dim.tot(), &input_h, &data); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = net->input_dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - net->infer(dim1, data); - TIMER_STOP - dim1.print(); - } - for(int i=0; i<3; i++) cudnn_out[i] = yolo[i]->dstData; - - - tk::dnn::dataDim_t dim2 = net->input_dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - for(int i=0; i<3; i++) rt_out[i] = (dnnType*)netRT.buffersRT[i+1]; - - int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; - for(int i=0; i<3; i++) { - printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); - dnnType *out, *out_h; - int odim = out_dim[i].tot(); - readBinaryFile(output_bins[i], odim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - } - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo3_512/yolo3_512.cpp b/tests/yolo3_512/yolo3_512.cpp deleted file mode 100644 index 5e796c1..0000000 --- a/tests/yolo3_512/yolo3_512.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include -#include -#include "tkdnn.h" - -int main() { - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); - tk::dnn::Network net(dim); - - // create yolo3 model - std::string bin_path = "yolo3_512"; - downloadWeightsifDoNotExist("yolo3_512/layers/input.bin", bin_path, "https://cloud.hipert.unimore.it/s/RGecMeGLD4cXEWL/download"); - int classes = 80; - tk::dnn::Yolo *yolo [3]; - #include "models/Yolo3.h" - - - - // fill classes names - for(int i=0; i<3; i++) { - yolo[i]->classesNames = {"person" , "bicycle" , "car" , "motorbike" , "aeroplane" , "bus" , "train" , "truck" , "boat" , "traffic light" , "fire hydrant" , "stop sign" , "parking meter" , "bench" , "bird" , "cat" , "dog" , "horse" , "sheep" , "cow" , "elephant" , "bear" , "zebra" , "giraffe" , "backpack" , "umbrella" , "handbag" , "tie" , "suitcase" , "frisbee" , "skis" , "snowboard" , "sports ball" , "kite" , "baseball bat" , "baseball glove" , "skateboard" , "surfboard" , "tennis racket" , "bottle" , "wine glass" , "cup" , "fork" , "knife" , "spoon" , "bowl" , "banana" , "apple" , "sandwich" , "orange" , "broccoli" , "carrot" , "hot dog" , "pizza" , "donut" , "cake" , "chair" , "sofa" , "pottedplant" , "bed" , "diningtable" , "toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" , "cell phone" , "microwave" , "oven" , "toaster" , "sink" , "refrigerator" , "book" , "clock" , "vase" , "scissors" , "teddy bear" , "hair drier" , "toothbrush"}; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_512")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - for(int i=0; i<3; i++) cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for(int i=0; i<3; i++) yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for(int j=0; j 0) - cl = c; - } - std::cout< -#include -#include "tkdnn.h" - -int main() { - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); - tk::dnn::Network net(dim); - - // create yolo3 model - std::string bin_path = "yolo3_512tp"; - // downloadWeightsifDoNotExist("yolo3_512tp/layers/input.bin", bin_path, ); - int classes = 3; - tk::dnn::Yolo *yolo [3]; - #include "models/Yolo3.h" - - // fill classes names - for(int i=0; i<3; i++) { - yolo[i]->classesNames = {"Dent", "Wrinkle", "UnsealedFlaps"}; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_512tp")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - for(int i=0; i<3; i++) cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for(int i=0; i<3; i++) yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for(int j=0; j 0) - cl = c; - } - std::cout< -#include -#include "tkdnn.h" - -int main() { - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 320, 544, 1); - tk::dnn::Network net(dim); - - // create yolo3 model - std::string bin_path = "yolo3_berkeley"; - downloadWeightsifDoNotExist("yolo3_berkeley/layers/input.bin", bin_path, "https://cloud.hipert.unimore.it/s/o5cHa4AjTKS64oD/download"); - int classes = 10; - tk::dnn::Yolo *yolo [3]; - #include "models/Yolo3.h" - - - - // fill classes names - for(int i=0; i<3; i++) { - yolo[i]->classesNames = {"person", "car", "truck", "bus", "motor", "bike", "rider", "traffic light", "traffic sign", "train"}; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_berkeley")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - for(int i=0; i<3; i++) cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for(int i=0; i<3; i++) yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for(int j=0; j 0) - cl = c; - } - std::cout< -#include -#include "tkdnn.h" - -int main() { - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 416, 416, 1); - tk::dnn::Network net(dim); - - // create yolo3 model - std::string bin_path = "yolo3_coco4"; - downloadWeightsifDoNotExist("yolo3_coco4/layers/input.bin", bin_path, "https://cloud.hipert.unimore.it/s/o27NDzSAartbyc4/download"); - int classes = 4; - tk::dnn::Yolo *yolo [3]; - #include "models/Yolo3.h" - - // fill classes names - for(int i=0; i<3; i++) { - yolo[i]->classesNames = {"person" , "bicycle" , "car" , "motorbike" }; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_coco4")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - for(int i=0; i<3; i++) cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for(int i=0; i<3; i++) yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for(int j=0; j 0) - cl = c; - } - std::cout< -#include -#include "tkdnn.h" - - -int main() { - - // Network layout - tk::dnn::dataDim_t dim(1, 1, 320, 544, 1); - tk::dnn::Network net(dim); - - // create yolo3 model - std::string bin_path = "yolo3_flir"; - downloadWeightsifDoNotExist("yolo3_flir/layers/input.bin", bin_path, "https://cloud.hipert.unimore.it/s/62DECncmF6bMMiH/download"); - - int classes = 3; - tk::dnn::Yolo *yolo [3]; - #include "models/Yolo3.h" - - // fill classes names - for(int i=0; i<3; i++) { - yolo[i]->classesNames = {"person", "bike", "car"}; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_flir")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - for(int i=0; i<3; i++) cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for(int i=0; i<3; i++) yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for(int j=0; j 0) - cl = c; - } - std::cout< -// #include -#include "tkdnn.h" - -const char *input_bin = "yolo3_tiny/layers/input.bin"; -const char *c0_bin = "yolo3_tiny/layers/c0.bin"; -const char *c2_bin = "yolo3_tiny/layers/c2.bin"; -const char *c4_bin = "yolo3_tiny/layers/c4.bin"; -const char *c6_bin = "yolo3_tiny/layers/c6.bin"; -const char *c8_bin = "yolo3_tiny/layers/c8.bin"; -const char *c10_bin = "yolo3_tiny/layers/c10.bin"; -const char *c12_bin = "yolo3_tiny/layers/c12.bin"; -const char *c13_bin = "yolo3_tiny/layers/c13.bin"; -const char *c14_bin = "yolo3_tiny/layers/c14.bin"; -const char *c15_bin = "yolo3_tiny/layers/c15.bin"; -const char *c18_bin = "yolo3_tiny/layers/c18.bin"; -const char *c21_bin = "yolo3_tiny/layers/c21.bin"; -const char *c22_bin = "yolo3_tiny/layers/c22.bin"; -const char *g16_bin = "yolo3_tiny/layers/g16.bin"; -const char *g23_bin = "yolo3_tiny/layers/g23.bin"; -// const char *output_bin = "yolo3_tiny/layers/output.bin"; - -const char *output_bin = "yolo3_tiny/debug/layer23_out.bin"; - -int main() { - - downloadWeightsifDoNotExist(input_bin, "yolo3_tiny", "https://cloud.hipert.unimore.it/s/LMcSHtWaLeps8yN/download"); - - int classes = 80; - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 416, 416, 1); - tk::dnn::Network net(dim); - - - tk::dnn::Conv2d c0 (&net, 16, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 32, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 64, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p5 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p7(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8(&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p9(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c10(&net, 512, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p11(&net, 2, 2, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - - tk::dnn::Conv2d c12(&net, 1024, 3, 3, 1, 1, 1, 1, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c13(&net, 256, 1, 1, 1, 1, 0, 0, c13_bin, true); - tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c14(&net, 512, 3, 3, 1, 1, 1, 1, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c15(&net, 255, 1, 1, 1, 1, 0, 0, c15_bin, false); - - tk::dnn::Yolo yolo0 (&net, classes, 2, g16_bin); - - tk::dnn::Layer *m17_layers[1] = { &a13 }; - tk::dnn::Route m17 (&net, m17_layers, 1); - tk::dnn::Conv2d c18(&net, 128, 1, 1, 1, 1, 0, 0, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u19 (&net, 2); - - tk::dnn::Layer *m20_layers[2] = { &u19, &a8 }; - tk::dnn::Route m20 (&net, m20_layers, 2); - - tk::dnn::Conv2d c21(&net, 256, 3, 3, 1, 1, 1, 1, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c22(&net, 255, 1, 1, 1, 1, 0, 0, c22_bin, false); - - tk::dnn::Yolo yolo1 (&net, classes, 2, g23_bin); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - // convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_tiny")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo3_tiny512/yolo3_tiny512.cpp b/tests/yolo3_tiny512/yolo3_tiny512.cpp deleted file mode 100644 index 38816f9..0000000 --- a/tests/yolo3_tiny512/yolo3_tiny512.cpp +++ /dev/null @@ -1,128 +0,0 @@ -#include -#include "tkdnn.h" - -const char *input_bin = "yolo3_tiny512/layers/input.bin"; -const char *c0_bin = "yolo3_tiny512/layers/c0.bin"; -const char *c2_bin = "yolo3_tiny512/layers/c2.bin"; -const char *c4_bin = "yolo3_tiny512/layers/c4.bin"; -const char *c6_bin = "yolo3_tiny512/layers/c6.bin"; -const char *c8_bin = "yolo3_tiny512/layers/c8.bin"; -const char *c10_bin = "yolo3_tiny512/layers/c10.bin"; -const char *c12_bin = "yolo3_tiny512/layers/c12.bin"; -const char *c13_bin = "yolo3_tiny512/layers/c13.bin"; -const char *c14_bin = "yolo3_tiny512/layers/c14.bin"; -const char *c15_bin = "yolo3_tiny512/layers/c15.bin"; -const char *c18_bin = "yolo3_tiny512/layers/c18.bin"; -const char *c21_bin = "yolo3_tiny512/layers/c21.bin"; -const char *c22_bin = "yolo3_tiny512/layers/c22.bin"; -const char *g16_bin = "yolo3_tiny512/layers/g16.bin"; -const char *g23_bin = "yolo3_tiny512/layers/g23.bin"; -// const char *output_bin = "yolo3_tiny512/layers/output.bin"; - -const char *output_bin = "yolo3_tiny512/debug/layer23_out.bin"; - -int main() { - - downloadWeightsifDoNotExist(input_bin, "yolo3_tiny512", "https://cloud.hipert.unimore.it/s/8Zt6bHwHADqP4JC/download"); - - int classes = 80; - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); - tk::dnn::Network net(dim); - - - tk::dnn::Conv2d c0 (&net, 16, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 32, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 64, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p5 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p7(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8(&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p9(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c10(&net, 512, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p11(&net, 2, 2, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - - tk::dnn::Conv2d c12(&net, 1024, 3, 3, 1, 1, 1, 1, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c13(&net, 256, 1, 1, 1, 1, 0, 0, c13_bin, true); - tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c14(&net, 512, 3, 3, 1, 1, 1, 1, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c15(&net, 255, 1, 1, 1, 1, 0, 0, c15_bin, false); - - tk::dnn::Yolo yolo0 (&net, classes, 2, g16_bin); - - tk::dnn::Layer *m17_layers[1] = { &a13 }; - tk::dnn::Route m17 (&net, m17_layers, 1); - tk::dnn::Conv2d c18(&net, 128, 1, 1, 1, 1, 0, 0, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u19 (&net, 2); - - tk::dnn::Layer *m20_layers[2] = { &u19, &a8 }; - tk::dnn::Route m20 (&net, m20_layers, 2); - - tk::dnn::Conv2d c21(&net, 256, 3, 3, 1, 1, 1, 1, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c22(&net, 255, 1, 1, 1, 1, 0, 0, c22_bin, false); - - tk::dnn::Yolo yolo1 (&net, classes, 2, g23_bin); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - // convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_tiny512")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo3_tiny512tp/yolo3_tiny512tp.cpp b/tests/yolo3_tiny512tp/yolo3_tiny512tp.cpp deleted file mode 100644 index 13a8b00..0000000 --- a/tests/yolo3_tiny512tp/yolo3_tiny512tp.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include -#include "tkdnn.h" - -const char *input_bin = "yolo3_tiny512tp/layers/input.bin"; -const char *c0_bin = "yolo3_tiny512tp/layers/c0.bin"; -const char *c2_bin = "yolo3_tiny512tp/layers/c2.bin"; -const char *c4_bin = "yolo3_tiny512tp/layers/c4.bin"; -const char *c6_bin = "yolo3_tiny512tp/layers/c6.bin"; -const char *c8_bin = "yolo3_tiny512tp/layers/c8.bin"; -const char *c10_bin = "yolo3_tiny512tp/layers/c10.bin"; -const char *c12_bin = "yolo3_tiny512tp/layers/c12.bin"; -const char *c13_bin = "yolo3_tiny512tp/layers/c13.bin"; -const char *c14_bin = "yolo3_tiny512tp/layers/c14.bin"; -const char *c15_bin = "yolo3_tiny512tp/layers/c15.bin"; -const char *c18_bin = "yolo3_tiny512tp/layers/c18.bin"; -const char *c21_bin = "yolo3_tiny512tp/layers/c21.bin"; -const char *c22_bin = "yolo3_tiny512tp/layers/c22.bin"; -const char *g16_bin = "yolo3_tiny512tp/layers/g16.bin"; -const char *g23_bin = "yolo3_tiny512tp/layers/g23.bin"; -// const char *output_bin = "yolo3_tiny512tp/layers/output.bin"; - -const char *output_bin = "yolo3_tiny512tp/debug/layer23_out.bin"; - -int main() { - - int classes = 3; - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); - tk::dnn::Network net(dim); - - - tk::dnn::Conv2d c0 (&net, 16, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 32, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 64, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p5 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p7(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8(&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p9(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c10(&net, 512, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p11(&net, 2, 2, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - - tk::dnn::Conv2d c12(&net, 1024, 3, 3, 1, 1, 1, 1, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c13(&net, 256, 1, 1, 1, 1, 0, 0, c13_bin, true); - tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c14(&net, 512, 3, 3, 1, 1, 1, 1, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c15(&net, 24, 1, 1, 1, 1, 0, 0, c15_bin, false); - - tk::dnn::Yolo yolo0 (&net, classes, 2, g16_bin); - - tk::dnn::Layer *m17_layers[1] = { &a13 }; - tk::dnn::Route m17 (&net, m17_layers, 1); - tk::dnn::Conv2d c18(&net, 128, 1, 1, 1, 1, 0, 0, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u19 (&net, 2); - - tk::dnn::Layer *m20_layers[2] = { &u19, &a8 }; - tk::dnn::Route m20 (&net, m20_layers, 2); - - tk::dnn::Conv2d c21(&net, 256, 3, 3, 1, 1, 1, 1, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c22(&net, 24, 1, 1, 1, 1, 0, 0, c22_bin, false); - - tk::dnn::Yolo yolo1 (&net, classes, 2, g23_bin); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - // convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_tiny512tp")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo3_tinyNM512/yolo3_tinyNM512.cpp b/tests/yolo3_tinyNM512/yolo3_tinyNM512.cpp deleted file mode 100644 index 7ffd603..0000000 --- a/tests/yolo3_tinyNM512/yolo3_tinyNM512.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include -#include "tkdnn.h" - -const char *input_bin = "yolo3_tinyNM512/layers/input.bin"; -const char *c0_bin = "yolo3_tinyNM512/layers/c0.bin"; -const char *c2_bin = "yolo3_tinyNM512/layers/c2.bin"; -const char *c4_bin = "yolo3_tinyNM512/layers/c4.bin"; -const char *c6_bin = "yolo3_tinyNM512/layers/c6.bin"; -const char *c8_bin = "yolo3_tinyNM512/layers/c8.bin"; -const char *c10_bin = "yolo3_tinyNM512/layers/c10.bin"; -const char *c11_bin = "yolo3_tinyNM512/layers/c11.bin"; -const char *c12_bin = "yolo3_tinyNM512/layers/c12.bin"; -const char *c13_bin = "yolo3_tinyNM512/layers/c13.bin"; -const char *c14_bin = "yolo3_tinyNM512/layers/c14.bin"; -const char *c17_bin = "yolo3_tinyNM512/layers/c17.bin"; -const char *c20_bin = "yolo3_tinyNM512/layers/c20.bin"; -const char *c21_bin = "yolo3_tinyNM512/layers/c21.bin"; -const char *g15_bin = "yolo3_tinyNM512/layers/g15.bin"; -const char *g22_bin = "yolo3_tinyNM512/layers/g22.bin"; -// const char *output_bin = "yolo3_tinyNM512/layers/output.bin"; - -const char *output_bin = "yolo3_tinyNM512/debug/layer22_out.bin"; - -int main() { - - // downloadWeightsifDoNotExist(input_bin, "yolo3_tinyNM512", "https://cloud.hipert.unimore.it/s/wRW9nmkibSe5HoS/download"); - - int classes = 80; - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); - tk::dnn::Network net(dim); - - - tk::dnn::Conv2d c0 (&net, 16, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 32, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 64, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p5 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p7(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8(&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p9(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c10(&net, 512, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c12(&net, 1024, 3, 3, 1, 1, 1, 1, c11_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c13(&net, 256, 1, 1, 1, 1, 0, 0, c12_bin, true); - tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c14(&net, 512, 3, 3, 1, 1, 1, 1, c13_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c15(&net, 255, 1, 1, 1, 1, 0, 0, c14_bin, false); - - tk::dnn::Yolo yolo0 (&net, classes, 2, g15_bin); - - tk::dnn::Layer *m17_layers[1] = { &a13 }; - tk::dnn::Route m17 (&net, m17_layers, 1); - tk::dnn::Conv2d c18(&net, 128, 1, 1, 1, 1, 0, 0, c17_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u19 (&net, 2); - - tk::dnn::Layer *m20_layers[2] = { &u19, &a8 }; - tk::dnn::Route m20 (&net, m20_layers, 2); - - tk::dnn::Conv2d c21(&net, 256, 3, 3, 1, 1, 1, 1, c20_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c22(&net, 255, 1, 1, 1, 1, 0, 0, c21_bin, false); - - tk::dnn::Yolo yolo1 (&net, classes, 2, g22_bin); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - // convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_tinyNM512")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo4/yolo4.cpp b/tests/yolo4/yolo4.cpp deleted file mode 100644 index 72ba205..0000000 --- a/tests/yolo4/yolo4.cpp +++ /dev/null @@ -1,666 +0,0 @@ -#include -#include -#include "tkdnn.h" - -int main() -{ - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 416, 416, 1); - tk::dnn::Network net(dim); - - // create yolo4 model - std::string bin_path = "yolo4"; - int classes = 80; - tk::dnn::Yolo *yolo[3]; - - std::string input_bin = bin_path + "/layers/input.bin"; - - std::vector output_bins = { - bin_path + "/debug/layer139_out.bin", - bin_path + "/debug/layer150_out.bin", - bin_path + "/debug/layer161_out.bin"}; - std::string c0_bin = bin_path + "/layers/c0.bin"; - std::string c1_bin = bin_path + "/layers/c1.bin"; - std::string c2_bin = bin_path + "/layers/c2.bin"; - std::string c3_bin = bin_path + "/layers/c3.bin"; - std::string c4_bin = bin_path + "/layers/c4.bin"; - std::string c5_bin = bin_path + "/layers/c5.bin"; - std::string c6_bin = bin_path + "/layers/c6.bin"; - std::string c7_bin = bin_path + "/layers/c7.bin"; - std::string c8_bin = bin_path + "/layers/c8.bin"; - std::string c10_bin = bin_path + "/layers/c10.bin"; - std::string c11_bin = bin_path + "/layers/c11.bin"; - std::string c12_bin = bin_path + "/layers/c12.bin"; - std::string c13_bin = bin_path + "/layers/c13.bin"; - std::string c14_bin = bin_path + "/layers/c14.bin"; - std::string c15_bin = bin_path + "/layers/c15.bin"; - std::string c16_bin = bin_path + "/layers/c16.bin"; - std::string c17_bin = bin_path + "/layers/c17.bin"; - std::string c18_bin = bin_path + "/layers/c18.bin"; - std::string c19_bin = bin_path + "/layers/c19.bin"; - std::string c20_bin = bin_path + "/layers/c20.bin"; - std::string c21_bin = bin_path + "/layers/c21.bin"; - std::string c23_bin = bin_path + "/layers/c23.bin"; - std::string c24_bin = bin_path + "/layers/c24.bin"; - std::string c25_bin = bin_path + "/layers/c25.bin"; - std::string c26_bin = bin_path + "/layers/c26.bin"; - std::string c27_bin = bin_path + "/layers/c27.bin"; - std::string c28_bin = bin_path + "/layers/c28.bin"; - std::string c29_bin = bin_path + "/layers/c29.bin"; - std::string c30_bin = bin_path + "/layers/c30.bin"; - std::string c31_bin = bin_path + "/layers/c31.bin"; - std::string c32_bin = bin_path + "/layers/c32.bin"; - std::string c33_bin = bin_path + "/layers/c33.bin"; - std::string c34_bin = bin_path + "/layers/c34.bin"; - std::string c35_bin = bin_path + "/layers/c35.bin"; - std::string c36_bin = bin_path + "/layers/c36.bin"; - std::string c37_bin = bin_path + "/layers/c37.bin"; - std::string c38_bin = bin_path + "/layers/c38.bin"; - std::string c39_bin = bin_path + "/layers/c39.bin"; - std::string c40_bin = bin_path + "/layers/c40.bin"; - std::string c41_bin = bin_path + "/layers/c41.bin"; - std::string c42_bin = bin_path + "/layers/c42.bin"; - std::string c43_bin = bin_path + "/layers/c43.bin"; - std::string c44_bin = bin_path + "/layers/c44.bin"; - std::string c45_bin = bin_path + "/layers/c45.bin"; - std::string c46_bin = bin_path + "/layers/c46.bin"; - std::string c47_bin = bin_path + "/layers/c47.bin"; - std::string c48_bin = bin_path + "/layers/c48.bin"; - std::string c49_bin = bin_path + "/layers/c49.bin"; - std::string c50_bin = bin_path + "/layers/c50.bin"; - std::string c51_bin = bin_path + "/layers/c51.bin"; - std::string c52_bin = bin_path + "/layers/c52.bin"; - std::string c53_bin = bin_path + "/layers/c53.bin"; - std::string c54_bin = bin_path + "/layers/c54.bin"; - std::string c55_bin = bin_path + "/layers/c55.bin"; - std::string c56_bin = bin_path + "/layers/c56.bin"; - std::string c57_bin = bin_path + "/layers/c57.bin"; - std::string c58_bin = bin_path + "/layers/c58.bin"; - std::string c59_bin = bin_path + "/layers/c59.bin"; - std::string c60_bin = bin_path + "/layers/c60.bin"; - std::string c61_bin = bin_path + "/layers/c61.bin"; - std::string c62_bin = bin_path + "/layers/c62.bin"; - std::string c63_bin = bin_path + "/layers/c63.bin"; - std::string c65_bin = bin_path + "/layers/c65.bin"; - std::string c66_bin = bin_path + "/layers/c66.bin"; - std::string c67_bin = bin_path + "/layers/c67.bin"; - std::string c68_bin = bin_path + "/layers/c68.bin"; - std::string c69_bin = bin_path + "/layers/c69.bin"; - std::string c70_bin = bin_path + "/layers/c70.bin"; - std::string c71_bin = bin_path + "/layers/c71.bin"; - std::string c72_bin = bin_path + "/layers/c72.bin"; - std::string c74_bin = bin_path + "/layers/c74.bin"; - std::string c75_bin = bin_path + "/layers/c75.bin"; - std::string c76_bin = bin_path + "/layers/c76.bin"; - std::string c77_bin = bin_path + "/layers/c77.bin"; - std::string c78_bin = bin_path + "/layers/c78.bin"; - std::string c80_bin = bin_path + "/layers/c80.bin"; - std::string c81_bin = bin_path + "/layers/c81.bin"; - std::string c82_bin = bin_path + "/layers/c82.bin"; - std::string c83_bin = bin_path + "/layers/c83.bin"; - std::string c85_bin = bin_path + "/layers/c85.bin"; - std::string c86_bin = bin_path + "/layers/c86.bin"; - std::string c87_bin = bin_path + "/layers/c87.bin"; - std::string c89_bin = bin_path + "/layers/c89.bin"; - std::string c90_bin = bin_path + "/layers/c90.bin"; - std::string c91_bin = bin_path + "/layers/c91.bin"; - std::string c92_bin = bin_path + "/layers/c92.bin"; - std::string c93_bin = bin_path + "/layers/c93.bin"; - std::string c94_bin = bin_path + "/layers/c94.bin"; - std::string c96_bin = bin_path + "/layers/c96.bin"; - std::string c97_bin = bin_path + "/layers/c97.bin"; - std::string c98_bin = bin_path + "/layers/c98.bin"; - std::string c99_bin = bin_path + "/layers/c99.bin"; - std::string c100_bin = bin_path + "/layers/c100.bin"; - std::string c101_bin = bin_path + "/layers/c101.bin"; - std::string c102_bin = bin_path + "/layers/c102.bin"; - std::string c103_bin = bin_path + "/layers/c103.bin"; - std::string c104_bin = bin_path + "/layers/c104.bin"; - std::string c105_bin = bin_path + "/layers/c105.bin"; - std::string c106_bin = bin_path + "/layers/c106.bin"; - std::string c107_bin = bin_path + "/layers/c107.bin"; - std::string c108_bin = bin_path + "/layers/c108.bin"; - std::string c109_bin = bin_path + "/layers/c109.bin"; - std::string c110_bin = bin_path + "/layers/c110.bin"; - std::string c111_bin = bin_path + "/layers/c111.bin"; - std::string c112_bin = bin_path + "/layers/c112.bin"; - std::string c113_bin = bin_path + "/layers/c113.bin"; - std::string c114_bin = bin_path + "/layers/c114.bin"; - std::string c115_bin = bin_path + "/layers/c115.bin"; - std::string c116_bin = bin_path + "/layers/c116.bin"; - std::string c117_bin = bin_path + "/layers/c117.bin"; - std::string c119_bin = bin_path + "/layers/c119.bin"; - std::string c120_bin = bin_path + "/layers/c120.bin"; - std::string c121_bin = bin_path + "/layers/c121.bin"; - std::string c122_bin = bin_path + "/layers/c122.bin"; - std::string c123_bin = bin_path + "/layers/c123.bin"; - std::string c124_bin = bin_path + "/layers/c124.bin"; - std::string c125_bin = bin_path + "/layers/c125.bin"; - std::string c126_bin = bin_path + "/layers/c126.bin"; - std::string c127_bin = bin_path + "/layers/c127.bin"; - std::string c128_bin = bin_path + "/layers/c128.bin"; - std::string c130_bin = bin_path + "/layers/c130.bin"; - std::string c131_bin = bin_path + "/layers/c131.bin"; - std::string c132_bin = bin_path + "/layers/c132.bin"; - std::string c133_bin = bin_path + "/layers/c133.bin"; - std::string c134_bin = bin_path + "/layers/c134.bin"; - std::string c135_bin = bin_path + "/layers/c135.bin"; - std::string c136_bin = bin_path + "/layers/c136.bin"; - std::string c137_bin = bin_path + "/layers/c137.bin"; - std::string c138_bin = bin_path + "/layers/c138.bin"; - std::string c141_bin = bin_path + "/layers/c141.bin"; - std::string c142_bin = bin_path + "/layers/c142.bin"; - std::string c143_bin = bin_path + "/layers/c143.bin"; - std::string c144_bin = bin_path + "/layers/c144.bin"; - std::string c145_bin = bin_path + "/layers/c145.bin"; - std::string c146_bin = bin_path + "/layers/c146.bin"; - std::string c147_bin = bin_path + "/layers/c147.bin"; - std::string c148_bin = bin_path + "/layers/c148.bin"; - std::string c149_bin = bin_path + "/layers/c149.bin"; - std::string c150_bin = bin_path + "/layers/c150.bin"; - std::string c151_bin = bin_path + "/layers/c151.bin"; - std::string c152_bin = bin_path + "/layers/c152.bin"; - std::string c153_bin = bin_path + "/layers/c153.bin"; - std::string c154_bin = bin_path + "/layers/c154.bin"; - std::string c155_bin = bin_path + "/layers/c155.bin"; - std::string c156_bin = bin_path + "/layers/c156.bin"; - std::string c157_bin = bin_path + "/layers/c157.bin"; - std::string c158_bin = bin_path + "/layers/c158.bin"; - std::string c159_bin = bin_path + "/layers/c159.bin"; - std::string c160_bin = bin_path + "/layers/c160.bin"; - std::string g139_bin = bin_path + "/layers/g139.bin"; - std::string g150_bin = bin_path + "/layers/g150.bin"; - std::string g161_bin = bin_path + "/layers/g161.bin"; - - - downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); - - tk::dnn::Conv2d c0(&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0(&net, tk::dnn::ACTIVATION_MISH); - - // downsample - tk::dnn::Conv2d c1(&net, 64, 3, 3, 2, 2, 1, 1, c1_bin, true); - tk::dnn::Activation a1(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c2(&net, 64, 1, 1, 1, 1, 0, 0, c2_bin, true); - tk::dnn::Activation a2(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r3_layers[1] = {&a1}; - tk::dnn::Route r3(&net, r3_layers, 1); - - tk::dnn::Conv2d c4(&net, 64, 1, 1, 1, 1, 0, 0, c4_bin, true); - tk::dnn::Activation a4(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c5(&net, 32, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c6(&net, 64, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s7(&net, &a4); - - tk::dnn::Conv2d c8(&net, 64, 1, 1, 1, 1, 0, 0, c8_bin, true); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r9_layers[2] = {&a8, &a2}; - tk::dnn::Route r9(&net, r9_layers, 2); - - tk::dnn::Conv2d c10(&net, 64, 1, 1, 1, 1, 0, 0, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_MISH); - - // downsample - tk::dnn::Conv2d c11(&net, 128, 3, 3, 2, 2, 1, 1, c11_bin, true); - tk::dnn::Activation a11(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c12(&net, 64, 1, 1, 1, 1, 0, 0, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r13_layers[1] = {&a11}; - tk::dnn::Route r13(&net, r13_layers, 1); - - tk::dnn::Conv2d c14(&net, 64, 1, 1, 1, 1, 0, 0, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c15(&net, 64, 1, 1, 1, 1, 0, 0, c15_bin, true); - tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c16(&net, 64, 3, 3, 1, 1, 1, 1, c16_bin, true); - tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s17(&net, &a14); - - tk::dnn::Conv2d c18(&net, 64, 1, 1, 1, 1, 0, 0, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c19(&net, 64, 3, 3, 1, 1, 1, 1, c19_bin, true); - tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s20(&net, &s17); - - tk::dnn::Conv2d c21(&net, 64, 1, 1, 1, 1, 0, 0, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r22_layers[2] = {&a21, &a12}; - tk::dnn::Route r22(&net, r22_layers, 2); - - tk::dnn::Conv2d c23(&net, 128, 1, 1, 1, 1, 0, 0, c23_bin, true); - tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c24(&net, 256, 3, 3, 2, 2, 1, 1, c24_bin, true); - tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c25(&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true); - tk::dnn::Activation a25(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r26_layers[1] = {&a24}; - tk::dnn::Route r26(&net, r26_layers, 1); - - tk::dnn::Conv2d c27(&net, 128, 1, 1, 1, 1, 0, 0, c27_bin, true); - tk::dnn::Activation a27(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c28(&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true); - tk::dnn::Activation a28(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c29(&net, 128, 3, 3, 1, 1, 1, 1, c29_bin, true); - tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s30(&net, &a27); - - tk::dnn::Conv2d c31(&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true); - tk::dnn::Activation a31(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c32(&net, 128, 3, 3, 1, 1, 1, 1, c32_bin, true); - tk::dnn::Activation a32(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s33(&net, &s30); - - tk::dnn::Conv2d c34(&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true); - tk::dnn::Activation a34(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c35(&net, 128, 3, 3, 1, 1, 1, 1, c35_bin, true); - tk::dnn::Activation a35(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s36(&net, &s33); - - tk::dnn::Conv2d c37(&net, 128, 1, 1, 1, 1, 0, 0, c37_bin, true); - tk::dnn::Activation a37(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c38(&net, 128, 3, 3, 1, 1, 1, 1, c38_bin, true); - tk::dnn::Activation a38(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s39(&net, &s36); - - tk::dnn::Conv2d c40(&net, 128, 1, 1, 1, 1, 0, 0, c40_bin, true); - tk::dnn::Activation a40(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c41(&net, 128, 3, 3, 1, 1, 1, 1, c41_bin, true); - tk::dnn::Activation a41(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s42(&net, &s39); - - tk::dnn::Conv2d c43(&net, 128, 1, 1, 1, 1, 0, 0, c43_bin, true); - tk::dnn::Activation a43(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c44(&net, 128, 3, 3, 1, 1, 1, 1, c44_bin, true); - tk::dnn::Activation a44(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s45(&net, &s42); - - tk::dnn::Conv2d c46(&net, 128, 1, 1, 1, 1, 0, 0, c46_bin, true); - tk::dnn::Activation a46(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c47(&net, 128, 3, 3, 1, 1, 1, 1, c47_bin, true); - tk::dnn::Activation a47(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s48(&net, &s45); - - tk::dnn::Conv2d c49(&net, 128, 1, 1, 1, 1, 0, 0, c49_bin, true); - tk::dnn::Activation a49(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c50(&net, 128, 3, 3, 1, 1, 1, 1, c50_bin, true); - tk::dnn::Activation a50(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s51(&net, &s48); - - tk::dnn::Conv2d c52(&net, 128, 1, 1, 1, 1, 0, 0, c52_bin, true); - tk::dnn::Activation a52(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r53_layers[2] = {&a52, &a25}; - tk::dnn::Route r53(&net, r53_layers, 2); - - tk::dnn::Conv2d c54(&net, 256, 1, 1, 1, 1, 0, 0, c54_bin, true); - tk::dnn::Activation a54(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c55(&net, 512, 3, 3, 2, 2, 1, 1, c55_bin, true); - tk::dnn::Activation a55(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c56(&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true); - tk::dnn::Activation a56(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r57_layers[1] = {&a55}; - tk::dnn::Route r57(&net, r57_layers, 1); - - tk::dnn::Conv2d c58(&net, 256, 1, 1, 1, 1, 0, 0, c58_bin, true); - tk::dnn::Activation a58(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c59(&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true); - tk::dnn::Activation a59(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c60(&net, 256, 3, 3, 1, 1, 1, 1, c60_bin, true); - tk::dnn::Activation a60(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s61(&net, &a58); - - tk::dnn::Conv2d c62(&net, 256, 1, 1, 1, 1, 0, 0, c62_bin, true); - tk::dnn::Activation a62(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c63(&net, 256, 3, 3, 1, 1, 1, 1, c63_bin, true); - tk::dnn::Activation a63(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s64(&net, &s61); - - tk::dnn::Conv2d c65(&net, 256, 1, 1, 1, 1, 0, 0, c65_bin, true); - tk::dnn::Activation a65(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c66(&net, 256, 3, 3, 1, 1, 1, 1, c66_bin, true); - tk::dnn::Activation a66(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s67(&net, &s64); - - tk::dnn::Conv2d c68(&net, 256, 1, 1, 1, 1, 0, 0, c68_bin, true); - tk::dnn::Activation a68(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c69(&net, 256, 3, 3, 1, 1, 1, 1, c69_bin, true); - tk::dnn::Activation a69(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s70(&net, &s67); - - tk::dnn::Conv2d c71(&net, 256, 1, 1, 1, 1, 0, 0, c71_bin, true); - tk::dnn::Activation a71(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c72(&net, 256, 3, 3, 1, 1, 1, 1, c72_bin, true); - tk::dnn::Activation a72(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s73(&net, &s70); - - tk::dnn::Conv2d c74(&net, 256, 1, 1, 1, 1, 0, 0, c74_bin, true); - tk::dnn::Activation a74(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c75(&net, 256, 3, 3, 1, 1, 1, 1, c75_bin, true); - tk::dnn::Activation a75(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s76(&net, &s73); - - tk::dnn::Conv2d c77(&net, 256, 1, 1, 1, 1, 0, 0, c77_bin, true); - tk::dnn::Activation a77(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c78(&net, 256, 3, 3, 1, 1, 1, 1, c78_bin, true); - tk::dnn::Activation a78(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s79(&net, &s76); - - tk::dnn::Conv2d c80(&net, 256, 1, 1, 1, 1, 0, 0, c80_bin, true); - tk::dnn::Activation a80(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c81(&net, 256, 3, 3, 1, 1, 1, 1, c81_bin, true); - tk::dnn::Activation a81(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s82(&net, &s79); - - tk::dnn::Conv2d c83(&net, 256, 1, 1, 1, 1, 0, 0, c83_bin, true); - tk::dnn::Activation a83(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r84_layers[2] = {&a83, &a56}; - tk::dnn::Route r84(&net, r84_layers, 2); - - tk::dnn::Conv2d c85(&net, 512, 1, 1, 1, 1, 0, 0, c85_bin, true); - tk::dnn::Activation a85(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c86(&net, 1024, 3, 3, 2, 2, 1, 1, c86_bin, true); - tk::dnn::Activation a86(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c87(&net, 512, 1, 1, 1, 1, 0, 0, c87_bin, true); - tk::dnn::Activation a87(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r88_layers[1] = {&a86}; - tk::dnn::Route r88(&net, r88_layers, 1); - - tk::dnn::Conv2d c89(&net, 512, 1, 1, 1, 1, 0, 0, c89_bin, true); - tk::dnn::Activation a89(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c90(&net, 512, 1, 1, 1, 1, 0, 0, c90_bin, true); - tk::dnn::Activation a90(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c91(&net, 512, 3, 3, 1, 1, 1, 1, c91_bin, true); - tk::dnn::Activation a91(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s92(&net, &a89); - - tk::dnn::Conv2d c93(&net, 512, 1, 1, 1, 1, 0, 0, c93_bin, true); - tk::dnn::Activation a93(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c94(&net, 512, 3, 3, 1, 1, 1, 1, c94_bin, true); - tk::dnn::Activation a94(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s95(&net, &s92); - - tk::dnn::Conv2d c96(&net, 512, 1, 1, 1, 1, 0, 0, c96_bin, true); - tk::dnn::Activation a96(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c97(&net, 512, 3, 3, 1, 1, 1, 1, c97_bin, true); - tk::dnn::Activation a97(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s98(&net, &s95); - - tk::dnn::Conv2d c99(&net, 512, 1, 1, 1, 1, 0, 0, c99_bin, true); - tk::dnn::Activation a99(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c100(&net, 512, 3, 3, 1, 1, 1, 1, c100_bin, true); - tk::dnn::Activation a100(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s101(&net, &s98); - - tk::dnn::Conv2d c102(&net, 512, 1, 1, 1, 1, 0, 0, c102_bin, true); - tk::dnn::Activation a102(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r103_layers[2] = {&a102, &a87}; - tk::dnn::Route r103(&net, r103_layers, 2); - - tk::dnn::Conv2d c104(&net, 1024, 1, 1, 1, 1, 0, 0, c104_bin, true); - tk::dnn::Activation a104(&net, tk::dnn::ACTIVATION_MISH); - - - //################ - tk::dnn::Conv2d c105(&net, 512, 1, 1, 1, 1, 0, 0, c105_bin, true); - tk::dnn::Activation a105(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c106(&net, 1024, 3, 3, 1, 1, 1, 1, c106_bin, true); - tk::dnn::Activation a106(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c107(&net, 512, 1, 1, 1, 1, 0, 0, c107_bin, true); - tk::dnn::Activation a107(&net, tk::dnn::ACTIVATION_LEAKY); - - //SPP - tk::dnn::Pooling p108(&net, 5, 5, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r109_layers[1] = {&a107}; - tk::dnn::Route r109(&net, r109_layers, 1); - - tk::dnn::Pooling p110(&net, 9, 9, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r111_layers[1] = {&a107}; - tk::dnn::Route r111(&net, r111_layers, 1); - - tk::dnn::Pooling p112(&net, 13, 13, 1, 1, 12, 12, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r113_layers[4] = {&p112, &p110, &p108, &a107}; - tk::dnn::Route r113(&net, r113_layers, 4); - //END SPP - - tk::dnn::Conv2d c114(&net, 512, 1, 1, 1, 1, 0, 0, c114_bin, true); - tk::dnn::Activation a114(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c115(&net, 1024, 3, 3, 1, 1, 1, 1, c115_bin, true); - tk::dnn::Activation a115(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c116(&net, 512, 1, 1, 1, 1, 0, 0, c116_bin, true); - tk::dnn::Activation a116(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c117(&net, 256, 1, 1, 1, 1, 0, 0, c117_bin, true); - tk::dnn::Activation a117(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u118(&net, 2); - tk::dnn::Layer *r119_layers[1] = {&a85}; - tk::dnn::Route r119(&net, r119_layers, 1); - tk::dnn::Conv2d c120(&net, 256, 1, 1, 1, 1, 0, 0, c120_bin, true); - tk::dnn::Activation a120(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r121_layers[2] = {&a120,&u118}; - tk::dnn::Route r121(&net, r121_layers, 2); - - tk::dnn::Conv2d c122(&net, 256, 1, 1, 1, 1, 0, 0, c122_bin, true); - tk::dnn::Activation a122(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c123(&net, 512, 3, 3, 1, 1, 1, 1, c123_bin, true); - tk::dnn::Activation a123(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c124(&net, 256, 1, 1, 1, 1, 0, 0, c124_bin, true); - tk::dnn::Activation a124(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c125(&net, 512, 3, 3, 1, 1, 1, 1, c125_bin, true); - tk::dnn::Activation a125(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c126(&net, 256, 1, 1, 1, 1, 0, 0, c126_bin, true); - tk::dnn::Activation a126(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c127(&net, 128, 1, 1, 1, 1, 0, 0, c127_bin, true); - tk::dnn::Activation a127(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u128(&net, 2); - tk::dnn::Layer *r129_layers[1] = {&a54}; - tk::dnn::Route r129(&net, r129_layers, 1); - tk::dnn::Conv2d c130(&net, 128, 1, 1, 1, 1, 0, 0, c130_bin, true); - tk::dnn::Activation a130(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r131_layers[2] = {&a130,&u128}; - tk::dnn::Route r131(&net, r131_layers, 2); - - - tk::dnn::Conv2d c132(&net, 128, 1, 1, 1, 1, 0, 0, c132_bin, true); - tk::dnn::Activation a132(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c133(&net, 256, 3, 3, 1, 1, 1, 1, c133_bin, true); - tk::dnn::Activation a133(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c134(&net, 128, 1, 1, 1, 1, 0, 0, c134_bin, true); - tk::dnn::Activation a134(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c135(&net, 256, 3, 3, 1, 1, 1, 1, c135_bin, true); - tk::dnn::Activation a135(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c136(&net, 128, 1, 1, 1, 1, 0, 0, c136_bin, true); - tk::dnn::Activation a136(&net, tk::dnn::ACTIVATION_LEAKY); - - - tk::dnn::Conv2d c137(&net, 256, 3, 3, 1, 1, 1, 1, c137_bin, true); - tk::dnn::Activation a137(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c138(&net, 255, 1, 1, 1, 1, 0, 0, c138_bin, false); - tk::dnn::Yolo yolo139(&net, classes, 3, g139_bin, 3, 1.2); - - tk::dnn::Layer *r140_layers[1] = {&a136}; - tk::dnn::Route r140(&net, r140_layers, 1); - tk::dnn::Conv2d c141(&net, 256, 3, 3, 2, 2, 1, 1, c141_bin, true); - tk::dnn::Activation a141(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r142_layers[2] = {&a141,&a126}; - tk::dnn::Route r142(&net, r142_layers, 2); - - tk::dnn::Conv2d c143(&net, 256, 1, 1, 1, 1, 0, 0, c143_bin, true); - tk::dnn::Activation a143(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c144(&net, 512, 3, 3, 1, 1, 1, 1, c144_bin, true); - tk::dnn::Activation a144(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c145(&net, 256, 1, 1, 1, 1, 0, 0, c145_bin, true); - tk::dnn::Activation a145(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c146(&net, 512, 3, 3, 1, 1, 1, 1, c146_bin, true); - tk::dnn::Activation a146(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c147(&net, 256, 1, 1, 1, 1, 0, 0, c147_bin, true); - tk::dnn::Activation a147(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c148(&net, 512, 3, 3, 1, 1, 1, 1, c148_bin, true); - tk::dnn::Activation a148(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c149(&net, 255, 1, 1, 1, 1, 0, 0, c149_bin, false); - tk::dnn::Yolo yolo150(&net, classes, 3, g150_bin, 3, 1.1); - - tk::dnn::Layer *r151_layers[1] = {&a147}; - tk::dnn::Route r151(&net, r151_layers, 1); - tk::dnn::Conv2d c152(&net, 512, 3, 3, 2, 2, 1, 1, c152_bin, true); - tk::dnn::Activation a152(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r153_layers[2] = {&a152,&a116}; - tk::dnn::Route r153(&net, r153_layers, 2); - - tk::dnn::Conv2d c154(&net, 512, 1, 1, 1, 1, 0, 0, c154_bin, true); - tk::dnn::Activation a154(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c155(&net, 1024, 3, 3, 1, 1, 1, 1, c155_bin, true); - tk::dnn::Activation a155(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c156(&net, 512, 1, 1, 1, 1, 0, 0, c156_bin, true); - tk::dnn::Activation a156(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c157(&net, 1024, 3, 3, 1, 1, 1, 1, c157_bin, true); - tk::dnn::Activation a157(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c158(&net, 512, 1, 1, 1, 1, 0, 0, c158_bin, true); - tk::dnn::Activation a158(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c159(&net, 1024, 3, 3, 1, 1, 1, 1, c159_bin, true); - tk::dnn::Activation a159(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c160(&net, 255, 1, 1, 1, 1, 0, 0, c160_bin, false); - tk::dnn::Yolo yolo161(&net, classes, 3, g161_bin, 3, 1.05); - - - - - - - yolo[0] = &yolo139; - yolo[1] = &yolo150; - yolo[2] = &yolo161; - - // fill classes names - for (int i = 0; i < 3; i++) - { - yolo[i]->classesNames = {"person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"}; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - // //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo4")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for (int i = 0; i < 3; i++) - out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); - { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - for (int i = 0; i < 3; i++) - cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for (int i = 0; i < 3; i++) - yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for (int j = 0; j < ndets; j++) - { - tk::dnn::Yolo::box b = dets[j].bbox; - int x0 = (b.x - b.w / 2.); - int x1 = (b.x + b.w / 2.); - int y0 = (b.y - b.h / 2.); - int y1 = (b.y + b.h / 2.); - - int cl = 0; - for (int c = 0; c < classes; ++c) - { - float prob = dets[j].prob[c]; - if (prob > 0) - cl = c; - } - std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; - } - TIMER_STOP - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); - { - dim2.print(); - TIMER_START - netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - for (int i = 0; i < 3; i++) - rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; - - int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; - for (int i = 0; i < 3; i++) - { - printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); - dnnType *out, *out_h; - int odim = out_dim[i].tot(); - readBinaryFile(output_bins[i], odim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - } - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo_224/yolo_224.cfg b/tests/yolo_224/yolo_224.cfg deleted file mode 100644 index dd9206c..0000000 --- a/tests/yolo_224/yolo_224.cfg +++ /dev/null @@ -1,258 +0,0 @@ -[net] -# Testing -#batch=1 -#subdivisions=1 -# Training - batch=64 - subdivisions=16 -width=224 -height=224 -channels=3 -momentum=0.9 -decay=0.0005 -angle=0 -saturation = 1.5 -exposure = 1.5 -hue=.1 - -learning_rate=0.001 -burn_in=1000 -max_batches = 500200 -policy=steps -steps=400000,450000 -scales=.1,.1 - -[convolutional] -batch_normalize=1 -filters=32 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=64 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=128 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=64 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=128 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - - -####### - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=leaky - -[route] -layers=-9 - -[convolutional] -batch_normalize=1 -size=1 -stride=1 -pad=1 -filters=64 -activation=leaky - -[reorg] -stride=2 - -[route] -layers=-1,-4 - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=leaky - -[convolutional] -size=1 -stride=1 -pad=1 -filters=425 -activation=linear - - -[region] -anchors = 0.57273, 0.677385, 1.87446, 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828 -bias_match=1 -classes=80 -coords=4 -num=5 -softmax=1 -jitter=.3 -rescore=1 - -object_scale=5 -noobject_scale=1 -class_scale=1 -coord_scale=1 - -absolute=1 -thresh = .6 -random=1 diff --git a/tests/yolo_224/yolo_224.cpp b/tests/yolo_224/yolo_224.cpp deleted file mode 100644 index c6d75ae..0000000 --- a/tests/yolo_224/yolo_224.cpp +++ /dev/null @@ -1,156 +0,0 @@ -#include -#include "tkdnn.h" - -const char *input_bin = "yolo_224/layers/input.bin"; -const char *c0_bin = "yolo_224/layers/c0.bin"; -const char *c2_bin = "yolo_224/layers/c2.bin"; -const char *c4_bin = "yolo_224/layers/c4.bin"; -const char *c5_bin = "yolo_224/layers/c5.bin"; -const char *c6_bin = "yolo_224/layers/c6.bin"; -const char *c8_bin = "yolo_224/layers/c8.bin"; -const char *c9_bin = "yolo_224/layers/c9.bin"; -const char *c10_bin = "yolo_224/layers/c10.bin"; -const char *c12_bin = "yolo_224/layers/c12.bin"; -const char *c13_bin = "yolo_224/layers/c13.bin"; -const char *c14_bin = "yolo_224/layers/c14.bin"; -const char *c15_bin = "yolo_224/layers/c15.bin"; -const char *c16_bin = "yolo_224/layers/c16.bin"; -const char *c18_bin = "yolo_224/layers/c18.bin"; -const char *c19_bin = "yolo_224/layers/c19.bin"; -const char *c20_bin = "yolo_224/layers/c20.bin"; -const char *c21_bin = "yolo_224/layers/c21.bin"; -const char *c22_bin = "yolo_224/layers/c22.bin"; -const char *c23_bin = "yolo_224/layers/c23.bin"; -const char *c24_bin = "yolo_224/layers/c24.bin"; -const char *c26_bin = "yolo_224/layers/c26.bin"; -const char *c29_bin = "yolo_224/layers/c29.bin"; -const char *c30_bin = "yolo_224/layers/c30.bin"; -const char *g31_bin = "yolo_224/layers/g31.bin"; -const char *output_bin = "yolo_224/layers/output.bin"; - -int main() { - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 224, 224, 1); - tk::dnn::Network net(dim); - - tk::dnn::Conv2d c0 (&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 64, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 128, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c5 (&net, 64, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p7 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8 (&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c9 (&net, 128, 1, 1, 1, 1, 0, 0, c9_bin, true); - tk::dnn::Activation a9 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c10(&net, 256, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p11(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c12(&net, 512, 3, 3, 1, 1, 1, 1, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c13(&net, 256, 1, 1, 1, 1, 0, 0, c13_bin, true); - tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c14(&net, 512, 3, 3, 1, 1, 1, 1, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c15(&net, 256, 1, 1, 1, 1, 0, 0, c15_bin, true); - tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c16(&net, 512, 3, 3, 1, 1, 1, 1, c16_bin, true); - tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p17(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c19(&net, 512, 1, 1, 1, 1, 0, 0, c19_bin, true); - tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true); - tk::dnn::Activation a20(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c21(&net, 512, 1, 1, 1, 1, 0, 0, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true); - tk::dnn::Activation a22(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true); - tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true); - tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *m25_layers[1] = { &a16 }; - tk::dnn::Route m25(&net, m25_layers, 1); - tk::dnn::Conv2d c26(&net, 64, 1, 1, 1, 1, 0, 0, c26_bin, true); - tk::dnn::Activation a26(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Reorg r27(&net, 2); - - tk::dnn::Layer *m28_layers[2] = { &r27, &a24 }; - tk::dnn::Route m28(&net, m28_layers, 2); - - tk::dnn::Conv2d c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true); - tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c30(&net, 425, 1, 1, 1, 1, 0, 0, c30_bin, false); - tk::dnn::Region g31(&net, 80, 4, 5); - - tk::dnn::RegionInterpret rI(dim, g31.output_dim, 80, 4, 5, 0.6f, g31_bin); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo_224")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - - // std::cout<<"\n\nDetected objects: \n"; - // dnnType *output_h = new dnnType[rI.output_dim.tot()]; - // checkCuda(cudaMemcpy(output_h, out_data2, - // rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost)); - // rI.interpretData(output_h); - // rI.showImageResult(input_h); - - - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo_berkeley/yolo_berkeley.cpp b/tests/yolo_berkeley/yolo_berkeley.cpp deleted file mode 100644 index 1d40c15..0000000 --- a/tests/yolo_berkeley/yolo_berkeley.cpp +++ /dev/null @@ -1,156 +0,0 @@ -#include -#include "tkdnn.h" - -const char *input_bin = "yolo_berkeley/layers/input.bin"; -const char *c0_bin = "yolo_berkeley/layers/c0.bin"; -const char *c2_bin = "yolo_berkeley/layers/c2.bin"; -const char *c4_bin = "yolo_berkeley/layers/c4.bin"; -const char *c5_bin = "yolo_berkeley/layers/c5.bin"; -const char *c6_bin = "yolo_berkeley/layers/c6.bin"; -const char *c8_bin = "yolo_berkeley/layers/c8.bin"; -const char *c9_bin = "yolo_berkeley/layers/c9.bin"; -const char *c10_bin = "yolo_berkeley/layers/c10.bin"; -const char *c12_bin = "yolo_berkeley/layers/c12.bin"; -const char *c13_bin = "yolo_berkeley/layers/c13.bin"; -const char *c14_bin = "yolo_berkeley/layers/c14.bin"; -const char *c15_bin = "yolo_berkeley/layers/c15.bin"; -const char *c16_bin = "yolo_berkeley/layers/c16.bin"; -const char *c18_bin = "yolo_berkeley/layers/c18.bin"; -const char *c19_bin = "yolo_berkeley/layers/c19.bin"; -const char *c20_bin = "yolo_berkeley/layers/c20.bin"; -const char *c21_bin = "yolo_berkeley/layers/c21.bin"; -const char *c22_bin = "yolo_berkeley/layers/c22.bin"; -const char *c23_bin = "yolo_berkeley/layers/c23.bin"; -const char *c24_bin = "yolo_berkeley/layers/c24.bin"; -const char *c26_bin = "yolo_berkeley/layers/c26.bin"; -const char *c29_bin = "yolo_berkeley/layers/c29.bin"; -const char *c30_bin = "yolo_berkeley/layers/c30.bin"; -const char *g31_bin = "yolo_berkeley/layers/g31.bin"; -const char *output_bin = "yolo_berkeley/layers/output.bin"; - -int main() { - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 416, 736, 1); - tk::dnn::Network net(dim); - - tk::dnn::Conv2d c0 (&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 64, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 128, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c5 (&net, 64, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p7 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8 (&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c9 (&net, 128, 1, 1, 1, 1, 0, 0, c9_bin, true); - tk::dnn::Activation a9 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c10(&net, 256, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p11(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c12(&net, 512, 3, 3, 1, 1, 1, 1, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c13(&net, 256, 1, 1, 1, 1, 0, 0, c13_bin, true); - tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c14(&net, 512, 3, 3, 1, 1, 1, 1, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c15(&net, 256, 1, 1, 1, 1, 0, 0, c15_bin, true); - tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c16(&net, 512, 3, 3, 1, 1, 1, 1, c16_bin, true); - tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p17(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c19(&net, 512, 1, 1, 1, 1, 0, 0, c19_bin, true); - tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true); - tk::dnn::Activation a20(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c21(&net, 512, 1, 1, 1, 1, 0, 0, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true); - tk::dnn::Activation a22(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true); - tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true); - tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *m25_layers[1] = { &a16 }; - tk::dnn::Route m25(&net, m25_layers, 1); - tk::dnn::Conv2d c26(&net, 64, 1, 1, 1, 1, 0, 0, c26_bin, true); - tk::dnn::Activation a26(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Reorg r27(&net, 2); - - tk::dnn::Layer *m28_layers[2] = { &r27, &a24 }; - tk::dnn::Route m28(&net, m28_layers, 2); - - tk::dnn::Conv2d c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true); - tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c30(&net, 75, 1, 1, 1, 1, 0, 0, c30_bin, false); - tk::dnn::Region g31(&net, 10, 4, 5); - - tk::dnn::RegionInterpret rI(dim, g31.output_dim, 10, 4, 5, 0.3f, g31_bin); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo_berkeley")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - - // std::cout<<"\n\nDetected objects: \n"; - // dnnType *output_h = new dnnType[rI.output_dim.tot()]; - // checkCuda(cudaMemcpy(output_h, out_data2, - // rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost)); - // rI.interpretData(output_h); - // rI.showImageResult(input_h); - - - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo_berkeley/yolov2-voc-10-resize-test.cfg b/tests/yolo_berkeley/yolov2-voc-10-resize-test.cfg deleted file mode 100644 index 184b32c..0000000 --- a/tests/yolo_berkeley/yolov2-voc-10-resize-test.cfg +++ /dev/null @@ -1,259 +0,0 @@ -[net] -# Testing -batch=1 -subdivisions=1 -# Training -#batch=64 -#subdivisions=8 -height=416 -width=736 -channels=3 -momentum=0.9 -decay=0.0005 -angle=0 -saturation = 1.5 -exposure = 1.5 -hue=.1 - -learning_rate=0.001 -burn_in=1000 -max_batches = 80200 -policy=steps -steps=40000,60000 -scales=.1,.1 - -[convolutional] -batch_normalize=1 -filters=32 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=64 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=128 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=64 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=128 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - - -####### - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=leaky - -[route] -layers=-9 - -[convolutional] -batch_normalize=1 -size=1 -stride=1 -pad=1 -filters=64 -activation=leaky - -[reorg] -stride=2 - -[route] -layers=-1,-4 - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=leaky - -[convolutional] -size=1 -stride=1 -pad=1 -filters=75 -activation=linear - - -[region] -anchors = 0.4043,0.4167, 1.2109,1.1018, 2.7258,2.1215, 4.9477,3.9132, 7.9508,6.6806 -bias_match=1 -classes=10 -coords=4 -num=5 -softmax=1 -jitter=.3 -rescore=1 - -object_scale=5 -noobject_scale=1 -class_scale=1 -coord_scale=1 - -absolute=1 -thresh = .6 -random=0 -flip=1 diff --git a/tests/yolo_relu/yolo_relu.cfg b/tests/yolo_relu/yolo_relu.cfg deleted file mode 100644 index 0abab4e..0000000 --- a/tests/yolo_relu/yolo_relu.cfg +++ /dev/null @@ -1,258 +0,0 @@ -[net] -# Testing -#batch=1 -#subdivisions=1 -# Training - batch=64 - subdivisions=16 -width=608 -height=608 -channels=3 -momentum=0.9 -decay=0.0005 -angle=0 -saturation = 1.5 -exposure = 1.5 -hue=.1 - -learning_rate=0.001 -burn_in=1000 -max_batches = 500200 -policy=steps -steps=400000,450000 -scales=.1,.1 - -[convolutional] -batch_normalize=1 -filters=32 -size=3 -stride=1 -pad=1 -activation=relu - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=64 -size=3 -stride=1 -pad=1 -activation=relu - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=128 -size=3 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=64 -size=1 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=128 -size=3 -stride=1 -pad=1 -activation=relu - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=relu - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=relu - -[maxpool] -size=2 -stride=2 - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=relu - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=relu - - -####### - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=relu - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=relu - -[route] -layers=-9 - -[convolutional] -batch_normalize=1 -size=1 -stride=1 -pad=1 -filters=64 -activation=relu - -[reorg] -stride=2 - -[route] -layers=-1,-4 - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=relu - -[convolutional] -size=1 -stride=1 -pad=1 -filters=425 -activation=linear - - -[region] -anchors = 0.57273, 0.677385, 1.87446, 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828 -bias_match=1 -classes=80 -coords=4 -num=5 -softmax=1 -jitter=.3 -rescore=1 - -object_scale=5 -noobject_scale=1 -class_scale=1 -coord_scale=1 - -absolute=1 -thresh = .6 -random=1 diff --git a/tests/yolo_relu/yolo_relu.cpp b/tests/yolo_relu/yolo_relu.cpp deleted file mode 100644 index 30f9f3e..0000000 --- a/tests/yolo_relu/yolo_relu.cpp +++ /dev/null @@ -1,155 +0,0 @@ -#include -#include "tkdnn.h" - -const char *input_bin = "yolo_relu/layers/input.bin"; -const char *c0_bin = "yolo_relu/layers/c0.bin"; -const char *c2_bin = "yolo_relu/layers/c2.bin"; -const char *c4_bin = "yolo_relu/layers/c4.bin"; -const char *c5_bin = "yolo_relu/layers/c5.bin"; -const char *c6_bin = "yolo_relu/layers/c6.bin"; -const char *c8_bin = "yolo_relu/layers/c8.bin"; -const char *c9_bin = "yolo_relu/layers/c9.bin"; -const char *c10_bin = "yolo_relu/layers/c10.bin"; -const char *c12_bin = "yolo_relu/layers/c12.bin"; -const char *c13_bin = "yolo_relu/layers/c13.bin"; -const char *c14_bin = "yolo_relu/layers/c14.bin"; -const char *c15_bin = "yolo_relu/layers/c15.bin"; -const char *c16_bin = "yolo_relu/layers/c16.bin"; -const char *c18_bin = "yolo_relu/layers/c18.bin"; -const char *c19_bin = "yolo_relu/layers/c19.bin"; -const char *c20_bin = "yolo_relu/layers/c20.bin"; -const char *c21_bin = "yolo_relu/layers/c21.bin"; -const char *c22_bin = "yolo_relu/layers/c22.bin"; -const char *c23_bin = "yolo_relu/layers/c23.bin"; -const char *c24_bin = "yolo_relu/layers/c24.bin"; -const char *c26_bin = "yolo_relu/layers/c26.bin"; -const char *c29_bin = "yolo_relu/layers/c29.bin"; -const char *c30_bin = "yolo_relu/layers/c30.bin"; -const char *g31_bin = "yolo_relu/layers/g31.bin"; -const char *output_bin = "yolo_relu/layers/output.bin"; - -int main() { - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 608, 608, 1); - tk::dnn::Network net(dim); - - tk::dnn::Conv2d c0 (&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 64, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 128, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c5 (&net, 64, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5 (&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Pooling p7 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8 (&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8 (&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c9 (&net, 128, 1, 1, 1, 1, 0, 0, c9_bin, true); - tk::dnn::Activation a9 (&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c10(&net, 256, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Pooling p11(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c12(&net, 512, 3, 3, 1, 1, 1, 1, c12_bin, true); - tk::dnn::Activation a12(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c13(&net, 256, 1, 1, 1, 1, 0, 0, c13_bin, true); - tk::dnn::Activation a13(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c14(&net, 512, 3, 3, 1, 1, 1, 1, c14_bin, true); - tk::dnn::Activation a14(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c15(&net, 256, 1, 1, 1, 1, 0, 0, c15_bin, true); - tk::dnn::Activation a15(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c16(&net, 512, 3, 3, 1, 1, 1, 1, c16_bin, true); - tk::dnn::Activation a16(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Pooling p17(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true); - tk::dnn::Activation a18(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c19(&net, 512, 1, 1, 1, 1, 0, 0, c19_bin, true); - tk::dnn::Activation a19(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true); - tk::dnn::Activation a20(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c21(&net, 512, 1, 1, 1, 1, 0, 0, c21_bin, true); - tk::dnn::Activation a21(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true); - tk::dnn::Activation a22(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true); - tk::dnn::Activation a23(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true); - tk::dnn::Activation a24(&net, CUDNN_ACTIVATION_RELU); - - tk::dnn::Layer *m25_layers[1] = { &a16 }; - tk::dnn::Route m25(&net, m25_layers, 1); - tk::dnn::Conv2d c26(&net, 64, 1, 1, 1, 1, 0, 0, c26_bin, true); - tk::dnn::Activation a26(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Reorg r27(&net, 2); - - tk::dnn::Layer *m28_layers[2] = { &r27, &a24 }; - tk::dnn::Route m28(&net, m28_layers, 2); - - tk::dnn::Conv2d c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true); - tk::dnn::Activation a29(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d c30(&net, 425, 1, 1, 1, 1, 0, 0, c30_bin, false); - tk::dnn::Region g31(&net, 80, 4, 5); - - tk::dnn::RegionInterpret rI(dim, g31.output_dim, 80, 4, 5, 0.3f, g31_bin); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo_relu")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - - // std::cout<<"\n\nDetected objects: \n"; - // dnnType *output_h = new dnnType[rI.output_dim.tot()]; - // checkCuda(cudaMemcpy(output_h, out_data2, - // rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost)); - // rI.interpretData(output_h, 608, 608); - // rI.showImageResult(input_h); - - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo_tiny/yolo_tiny.cpp b/tests/yolo_tiny/yolo_tiny.cpp deleted file mode 100644 index dd6857e..0000000 --- a/tests/yolo_tiny/yolo_tiny.cpp +++ /dev/null @@ -1,100 +0,0 @@ -#include -#include "tkdnn.h" - -const char *input_bin = "yolo_tiny/layers/input.bin"; -const char *c0_bin = "yolo_tiny/layers/c0.bin"; -const char *c2_bin = "yolo_tiny/layers/c2.bin"; -const char *c4_bin = "yolo_tiny/layers/c4.bin"; -const char *c5_bin = "yolo_tiny/layers/c5.bin"; -const char *c6_bin = "yolo_tiny/layers/c6.bin"; -const char *c8_bin = "yolo_tiny/layers/c8.bin"; -const char *c10_bin = "yolo_tiny/layers/c10.bin"; -const char *c11_bin = "yolo_tiny/layers/c11.bin"; -const char *c12_bin = "yolo_tiny/layers/c12.bin"; -const char *c13_bin = "yolo_tiny/layers/c13.bin"; -const char *g14_bin = "yolo_tiny/layers/g14.bin"; -const char *output_bin = "yolo_tiny/layers/output.bin"; - -int main() { - - downloadWeightsifDoNotExist(input_bin, "yolo_tiny", "https://cloud.hipert.unimore.it/s/m3orfJr8pGrN5mQ/download"); - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 416, 416, 1); - tk::dnn::Network net(dim); - - tk::dnn::Conv2d c0 (&net, 16, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 32, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 64, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p5 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p7(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8(&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p9(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c10(&net, 512, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c11(&net, 1024, 3, 3, 1, 1, 1, 1, c11_bin, true); - tk::dnn::Activation a11(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c12(&net, 512, 3, 3, 1, 1, 1, 1, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c13(&net, 425, 1, 1, 1, 1, 0, 0, c13_bin, false); - tk::dnn::Region g14(&net, 80, 4, 5); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo_tiny")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo_voc/yolo_voc.cpp b/tests/yolo_voc/yolo_voc.cpp deleted file mode 100644 index 868128b..0000000 --- a/tests/yolo_voc/yolo_voc.cpp +++ /dev/null @@ -1,156 +0,0 @@ -#include -#include "tkdnn.h" - -const char *input_bin = "yolo_voc/layers/input.bin"; -const char *c0_bin = "yolo_voc/layers/c0.bin"; -const char *c2_bin = "yolo_voc/layers/c2.bin"; -const char *c4_bin = "yolo_voc/layers/c4.bin"; -const char *c5_bin = "yolo_voc/layers/c5.bin"; -const char *c6_bin = "yolo_voc/layers/c6.bin"; -const char *c8_bin = "yolo_voc/layers/c8.bin"; -const char *c9_bin = "yolo_voc/layers/c9.bin"; -const char *c10_bin = "yolo_voc/layers/c10.bin"; -const char *c12_bin = "yolo_voc/layers/c12.bin"; -const char *c13_bin = "yolo_voc/layers/c13.bin"; -const char *c14_bin = "yolo_voc/layers/c14.bin"; -const char *c15_bin = "yolo_voc/layers/c15.bin"; -const char *c16_bin = "yolo_voc/layers/c16.bin"; -const char *c18_bin = "yolo_voc/layers/c18.bin"; -const char *c19_bin = "yolo_voc/layers/c19.bin"; -const char *c20_bin = "yolo_voc/layers/c20.bin"; -const char *c21_bin = "yolo_voc/layers/c21.bin"; -const char *c22_bin = "yolo_voc/layers/c22.bin"; -const char *c23_bin = "yolo_voc/layers/c23.bin"; -const char *c24_bin = "yolo_voc/layers/c24.bin"; -const char *c26_bin = "yolo_voc/layers/c26.bin"; -const char *c29_bin = "yolo_voc/layers/c29.bin"; -const char *c30_bin = "yolo_voc/layers/c30.bin"; -const char *g31_bin = "yolo_voc/layers/g31.bin"; -const char *output_bin = "yolo_voc/layers/output.bin"; - -int main() { - - downloadWeightsifDoNotExist(input_bin, "yolo_voc", "https://cloud.hipert.unimore.it/s/DJC5Fi2pEjfNDP9/download"); - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 416, 416, 1); - tk::dnn::Network net(dim); - - tk::dnn::Conv2d c0 (&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p1 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c2 (&net, 64, 3, 3, 1, 1, 1, 1, c2_bin, true); - tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p3 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c4 (&net, 128, 3, 3, 1, 1, 1, 1, c4_bin, true); - tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c5 (&net, 64, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c6 (&net, 128, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p7 (&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c8 (&net, 256, 3, 3, 1, 1, 1, 1, c8_bin, true); - tk::dnn::Activation a8 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c9 (&net, 128, 1, 1, 1, 1, 0, 0, c9_bin, true); - tk::dnn::Activation a9 (&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c10(&net, 256, 3, 3, 1, 1, 1, 1, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p11(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c12(&net, 512, 3, 3, 1, 1, 1, 1, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c13(&net, 256, 1, 1, 1, 1, 0, 0, c13_bin, true); - tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c14(&net, 512, 3, 3, 1, 1, 1, 1, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c15(&net, 256, 1, 1, 1, 1, 0, 0, c15_bin, true); - tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c16(&net, 512, 3, 3, 1, 1, 1, 1, c16_bin, true); - tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Pooling p17(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); - - tk::dnn::Conv2d c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c19(&net, 512, 1, 1, 1, 1, 0, 0, c19_bin, true); - tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true); - tk::dnn::Activation a20(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c21(&net, 512, 1, 1, 1, 1, 0, 0, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true); - tk::dnn::Activation a22(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true); - tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true); - tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Layer *m25_layers[1] = { &a16 }; - tk::dnn::Route m25(&net, m25_layers, 1); - tk::dnn::Conv2d c26(&net, 64, 1, 1, 1, 1, 0, 0, c26_bin, true); - tk::dnn::Activation a26(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Reorg r27(&net, 2); - - tk::dnn::Layer *m28_layers[2] = { &r27, &a24 }; - tk::dnn::Route m28(&net, m28_layers, 2); - - tk::dnn::Conv2d c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true); - tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c30(&net, 125, 1, 1, 1, 1, 0, 0, c30_bin, false); - tk::dnn::Region g31(&net, 20, 4, 5); - - tk::dnn::RegionInterpret rI(dim, g31.output_dim, 20, 4, 5, 0.6f, g31_bin); - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo_voc")); - - dnnType *out_data, *out_data2; // cudnn output, tensorRT output - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); { - dim1.print(); - TIMER_START - out_data = net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); { - dim2.print(); - TIMER_START - out_data2 = netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - printCenteredTitle(" CHECK RESULTS ", '=', 30); - dnnType *out, *out_h; - int out_dim = net.getOutputDim().tot(); - readBinaryFile(output_bin, out_dim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - int ret_cudnn = checkResult(out_dim, out_data, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - int ret_tensorrt = checkResult(out_dim, out_data2, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - int ret_cudnn_tensorrt = checkResult(out_dim, out_data, out_data2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - // std::cout<<"\n\nDetected objects: \n"; - // dnnType *output_h = new dnnType[rI.output_dim.tot()]; - // checkCuda(cudaMemcpy(output_h, out_data2, - // rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost)); - // rI.interpretData(output_h); - // rI.showImageResult(input_h); - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} -- 2.52.0 From 18794d52c28be700c79f5e7d927122802a8b12d2 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Mon, 1 Jun 2020 14:55:08 +0200 Subject: [PATCH 034/228] darknet parser tested --- include/tkDNN/DarknetParser.h | 47 ++++++++++++++----- include/tkDNN/Network.h | 1 + include/tkDNN/test.h | 5 ++ scripts/test_all_tests.sh | 3 +- src/Network.cpp | 8 +++- src/Region.cpp | 3 +- tests/darknet/cfg/yolo2tiny.cfg | 8 ++-- tests/darknet/csresnext50-panet-spp.cpp | 1 + .../csresnext50-panet-spp_berkeley.cpp | 4 +- tests/darknet/yolo2.cpp | 3 +- tests/darknet/yolo2_voc.cpp | 1 + tests/darknet/yolo2tiny.cpp | 6 ++- tests/darknet/yolo3.cpp | 3 +- tests/darknet/yolo3_512.cpp | 1 + tests/darknet/yolo3_berkeley.cpp | 3 +- tests/darknet/yolo3_coco4.cpp | 1 + tests/darknet/yolo3_flir.cpp | 1 + tests/darknet/yolo3tiny.cpp | 4 +- .../{yolo3tiny512.cpp => yolo3tiny_512.cpp} | 4 +- tests/darknet/yolo4.cpp | 1 + tests/darknet/yolo4_berkeley.cpp | 34 ++++++++++++++ 21 files changed, 112 insertions(+), 30 deletions(-) rename tests/darknet/{yolo3tiny512.cpp => yolo3tiny_512.cpp} (89%) create mode 100644 tests/darknet/yolo4_berkeley.cpp diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index 9008670..a918a5b 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -22,6 +22,7 @@ namespace tk { namespace dnn { int classes = 20; int num = 1; int pad = 0; + int coords = 4; float scale_xy = 1; std::vector layers; std::string activation = "linear"; @@ -102,9 +103,11 @@ namespace tk { namespace dnn { fields.classes = std::stoi(value); else if(name.find("num") != std::string::npos) fields.num = std::stoi(value); + else if(name.find("coords") != std::string::npos) + fields.coords = std::stoi(value); else if(name.find("groups") != std::string::npos) fields.groups = std::stoi(value); - else if(name.find("scale_xy") != std::string::npos) + else if(name.find("scale_x_y") != std::string::npos) fields.scale_xy = std::stof(value); else if(name.find("from") != std::string::npos) fields.layers.push_back(std::stof(value)); @@ -135,23 +138,25 @@ namespace tk { namespace dnn { if(f.pad == 1) { f.padding_x = f.padding_y = f.size_x /2; } - std::cout<<"Add layer: "< 0 && f.activation != "linear") { + tkdnnActivationMode_t act; + if(f.activation == "relu") act = tkdnnActivationMode_t(CUDNN_ACTIVATION_RELU); + else if(f.activation == "leaky") act = tk::dnn::ACTIVATION_LEAKY; + else if(f.activation == "mish") act = tk::dnn::ACTIVATION_MISH; + else { FatalError("activation not supported: " + f.activation); } + netLayers[netLayers.size()-1] = new tk::dnn::Activation(net, act); + }; } std::vector darknetReadNames(const std::string& names_file){ @@ -244,7 +265,7 @@ namespace tk { namespace dnn { // new type //std::cout<<"type: "< input_bins, std::vector if(net->layers[i]->final) outputs.push_back(net->layers[i]); } + // no final layers, set last as output + if(outputs.size() == 0) { + outputs.push_back(net->layers[net->num_layers-1]); + } + // check input if(input_bins.size() != 1) { diff --git a/scripts/test_all_tests.sh b/scripts/test_all_tests.sh index 1a621f7..b530606 100644 --- a/scripts/test_all_tests.sh +++ b/scripts/test_all_tests.sh @@ -78,9 +78,10 @@ do test_net yolo3_512 test_net yolo3tiny test_net csresnext50-panet-spp + #test_net csresnext50-panet-spp_berkeley test_net mobilenetv2ssd test_net yolo3tiny_512 - test_net yolo2tiny + #test_net yolo2tiny test_net mobilenetv2ssd512 test_net mnist test_net yolo2 diff --git a/src/Network.cpp b/src/Network.cpp index 6801ba3..9adcc48 100644 --- a/src/Network.cpp +++ b/src/Network.cpp @@ -59,12 +59,16 @@ Network::Network(dataDim_t input_dim) { } Network::~Network() { - for(int i=0; iclasses = classes; this->coords = coords; this->num = num; diff --git a/tests/darknet/cfg/yolo2tiny.cfg b/tests/darknet/cfg/yolo2tiny.cfg index 630a209..2884bb4 100644 --- a/tests/darknet/cfg/yolo2tiny.cfg +++ b/tests/darknet/cfg/yolo2tiny.cfg @@ -1,5 +1,5 @@ [net] - Training +# Training batch=64 subdivisions=8 # Testing @@ -90,9 +90,9 @@ stride=1 pad=1 activation=leaky -#[maxpool] -#size=2 -#stride=1 +[maxpool] +size=2 +stride=1 [convolutional] batch_normalize=1 diff --git a/tests/darknet/csresnext50-panet-spp.cpp b/tests/darknet/csresnext50-panet-spp.cpp index f2f66f6..1da95b2 100644 --- a/tests/darknet/csresnext50-panet-spp.cpp +++ b/tests/darknet/csresnext50-panet-spp.cpp @@ -27,6 +27,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/csresnext50-panet-spp_berkeley.cpp b/tests/darknet/csresnext50-panet-spp_berkeley.cpp index ca21399..47bbfd4 100644 --- a/tests/darknet/csresnext50-panet-spp_berkeley.cpp +++ b/tests/darknet/csresnext50-panet-spp_berkeley.cpp @@ -5,7 +5,7 @@ #include "DarknetParser.h" int main() { - std::string bin_path = "bdd-csresnext50-panet-spp"; + std::string bin_path = "csresnext50-panet-spp_berkeley"; std::vector input_bins = { bin_path + "/layers/input.bin" }; @@ -17,6 +17,7 @@ int main() { std::string wgs_path = bin_path + "/layers"; std::string cfg_path = "../tests/darknet/cfg/csresnext50-panet-spp_berkeley.cfg"; std::string name_path = "../tests/darknet/names/berkeley.names"; + // FIXME: wrong weights // downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s//download"); // parse darknet network @@ -27,6 +28,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo2.cpp b/tests/darknet/yolo2.cpp index e6b40a5..7a46c31 100644 --- a/tests/darknet/yolo2.cpp +++ b/tests/darknet/yolo2.cpp @@ -10,7 +10,7 @@ int main() { bin_path + "/layers/input.bin" }; std::vector output_bins = { - bin_path + "layers/output.bin" + bin_path + "/layers/output.bin" }; std::string wgs_path = bin_path + "/layers"; std::string cfg_path = "../tests/darknet/cfg/yolo2.cfg"; @@ -25,6 +25,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo2_voc.cpp b/tests/darknet/yolo2_voc.cpp index 2efcbcb..eab215b 100644 --- a/tests/darknet/yolo2_voc.cpp +++ b/tests/darknet/yolo2_voc.cpp @@ -25,6 +25,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo2tiny.cpp b/tests/darknet/yolo2tiny.cpp index fefdaa5..64faa36 100644 --- a/tests/darknet/yolo2tiny.cpp +++ b/tests/darknet/yolo2tiny.cpp @@ -10,12 +10,13 @@ int main() { bin_path + "/layers/input.bin" }; std::vector output_bins = { - bin_path + "layers/output.bin" + bin_path + "/layers/output.bin" }; std::string wgs_path = bin_path + "/layers"; std::string cfg_path = "../tests/darknet/cfg/yolo2tiny.cfg"; std::string name_path = "../tests/darknet/names/coco.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/nf4PJ3k8bxBETwL/download"); + // FIXME: wrong weights + //downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s//download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); @@ -25,6 +26,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo3.cpp b/tests/darknet/yolo3.cpp index b96d79a..ea53b84 100644 --- a/tests/darknet/yolo3.cpp +++ b/tests/darknet/yolo3.cpp @@ -26,7 +26,8 @@ int main() { //convert network to tensorRT tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); - int ret = testInference(input_bins, output_bins, net, netRT); + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo3_512.cpp b/tests/darknet/yolo3_512.cpp index 11a7839..a67c3a7 100644 --- a/tests/darknet/yolo3_512.cpp +++ b/tests/darknet/yolo3_512.cpp @@ -27,6 +27,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo3_berkeley.cpp b/tests/darknet/yolo3_berkeley.cpp index 6c3512c..a71fe83 100644 --- a/tests/darknet/yolo3_berkeley.cpp +++ b/tests/darknet/yolo3_berkeley.cpp @@ -16,7 +16,7 @@ int main() { }; std::string wgs_path = bin_path + "/layers"; std::string cfg_path = "../tests/darknet/cfg/yolo3_berkeley.cfg"; - std::string name_path = "../tests/darknet/names/barkeley.names"; + std::string name_path = "../tests/darknet/names/berkeley.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/o5cHa4AjTKS64oD/download"); // parse darknet network @@ -27,6 +27,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo3_coco4.cpp b/tests/darknet/yolo3_coco4.cpp index cf69a26..a651430 100644 --- a/tests/darknet/yolo3_coco4.cpp +++ b/tests/darknet/yolo3_coco4.cpp @@ -27,6 +27,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo3_flir.cpp b/tests/darknet/yolo3_flir.cpp index fd5c1d5..678f10f 100644 --- a/tests/darknet/yolo3_flir.cpp +++ b/tests/darknet/yolo3_flir.cpp @@ -27,6 +27,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo3tiny.cpp b/tests/darknet/yolo3tiny.cpp index 247b152..01fc6f9 100644 --- a/tests/darknet/yolo3tiny.cpp +++ b/tests/darknet/yolo3tiny.cpp @@ -10,7 +10,8 @@ int main() { bin_path + "/layers/input.bin" }; std::vector output_bins = { - bin_path + "debug/layer23_out.bin", + bin_path + "/debug/layer16_out.bin", + bin_path + "/debug/layer23_out.bin", }; std::string wgs_path = bin_path + "/layers"; std::string cfg_path = "../tests/darknet/cfg/yolo3tiny.cfg"; @@ -25,6 +26,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo3tiny512.cpp b/tests/darknet/yolo3tiny_512.cpp similarity index 89% rename from tests/darknet/yolo3tiny512.cpp rename to tests/darknet/yolo3tiny_512.cpp index 495033a..8153b0d 100644 --- a/tests/darknet/yolo3tiny512.cpp +++ b/tests/darknet/yolo3tiny_512.cpp @@ -10,7 +10,8 @@ int main() { bin_path + "/layers/input.bin" }; std::vector output_bins = { - bin_path + "debug/layer23_out.bin", + bin_path + "/debug/layer16_out.bin", + bin_path + "/debug/layer23_out.bin", }; std::string wgs_path = bin_path + "/layers"; std::string cfg_path = "../tests/darknet/cfg/yolo3tiny_512.cfg"; @@ -25,6 +26,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo4.cpp b/tests/darknet/yolo4.cpp index 70257d2..80b12ce 100644 --- a/tests/darknet/yolo4.cpp +++ b/tests/darknet/yolo4.cpp @@ -27,6 +27,7 @@ int main() { tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); delete net; delete netRT; return ret; diff --git a/tests/darknet/yolo4_berkeley.cpp b/tests/darknet/yolo4_berkeley.cpp new file mode 100644 index 0000000..db534b7 --- /dev/null +++ b/tests/darknet/yolo4_berkeley.cpp @@ -0,0 +1,34 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4_berkeley"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo4_berkeley.cfg"; + std::string name_path = "../tests/darknet/names/berkeley.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s//download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} -- 2.52.0 From a0e4e9f209641a2d43f0fbcf799445d91c8e1364 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 1 Jun 2020 15:17:53 +0200 Subject: [PATCH 035/228] Modify yolov4_berkeley link for download, clean to merge with master Signed-off-by: Micaela Verucchi --- CMakeLists.txt | 9 - demo/demo/map.cpp | 2 +- include/tkDNN/DetectionNN.h | 8 +- include/tkDNN/utils.h | 4 +- tests/yolo4/yolo4_320.cpp | 666 ------------------------ tests/yolo4/yolo4_512.cpp | 666 ------------------------ tests/yolo4/yolo4_608.cpp | 666 ------------------------ tests/yolo4_berkeley/yolo4_berkeley.cpp | 2 +- 8 files changed, 8 insertions(+), 2015 deletions(-) delete mode 100644 tests/yolo4/yolo4_320.cpp delete mode 100644 tests/yolo4/yolo4_512.cpp delete mode 100644 tests/yolo4/yolo4_608.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e80b87..ef593ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,15 +116,6 @@ target_link_libraries(test_yolo3_flir tkDNN) add_executable(test_yolo4 tests/yolo4/yolo4.cpp) target_link_libraries(test_yolo4 tkDNN) -add_executable(test_yolo4_320 tests/yolo4/yolo4_320.cpp) -target_link_libraries(test_yolo4_320 tkDNN) - -add_executable(test_yolo4_512 tests/yolo4/yolo4_512.cpp) -target_link_libraries(test_yolo4_512 tkDNN) - -add_executable(test_yolo4_608 tests/yolo4/yolo4_608.cpp) -target_link_libraries(test_yolo4_608 tkDNN) - add_executable(test_yolo4_berkeley tests/yolo4_berkeley/yolo4_berkeley.cpp) target_link_libraries(test_yolo4_berkeley tkDNN) diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index aefc834..60aa7fa 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -210,7 +210,7 @@ int main(int argc, char *argv[]) } if(write_coco_json){ - coco_json.seekp (coco_json.tellp()-2); + coco_json.seekp (coco_json.tellp() - std::streampos(2)); coco_json << "\n]\n"; coco_json.close(); } diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index 2e64be8..144ded5 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -14,7 +14,7 @@ #include "tkdnn.h" -#define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. +// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. #ifdef OPENCV_CUDACONTRIB #include @@ -104,7 +104,7 @@ class DetectionNN { FatalError("A batch size greater than nBatches cannot be used"); originalSize.clear(); - if(VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); { TIMER_START for(int bi=0; biinput_dim; dim.n = cur_batches; { - if(VERBOSE) dim.print(); + if(TKDNN_VERBOSE) dim.print(); TIMER_START netRT->infer(dim, input_d); TIMER_STOP - if(VERBOSE) dim.print(); + if(TKDNN_VERBOSE) dim.print(); stats.push_back(t_ns); if(save_times) *times< -#include -#include "tkdnn.h" - -int main() -{ - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 320, 320, 1); - tk::dnn::Network net(dim); - - // create yolo4_320 model - std::string bin_path = "yolo4_320"; - int classes = 80; - tk::dnn::Yolo *yolo[3]; - - std::string input_bin = bin_path + "/layers/input.bin"; - - std::vector output_bins = { - bin_path + "/debug/layer139_out.bin", - bin_path + "/debug/layer150_out.bin", - bin_path + "/debug/layer161_out.bin"}; - std::string c0_bin = bin_path + "/layers/c0.bin"; - std::string c1_bin = bin_path + "/layers/c1.bin"; - std::string c2_bin = bin_path + "/layers/c2.bin"; - std::string c3_bin = bin_path + "/layers/c3.bin"; - std::string c4_bin = bin_path + "/layers/c4.bin"; - std::string c5_bin = bin_path + "/layers/c5.bin"; - std::string c6_bin = bin_path + "/layers/c6.bin"; - std::string c7_bin = bin_path + "/layers/c7.bin"; - std::string c8_bin = bin_path + "/layers/c8.bin"; - std::string c10_bin = bin_path + "/layers/c10.bin"; - std::string c11_bin = bin_path + "/layers/c11.bin"; - std::string c12_bin = bin_path + "/layers/c12.bin"; - std::string c13_bin = bin_path + "/layers/c13.bin"; - std::string c14_bin = bin_path + "/layers/c14.bin"; - std::string c15_bin = bin_path + "/layers/c15.bin"; - std::string c16_bin = bin_path + "/layers/c16.bin"; - std::string c17_bin = bin_path + "/layers/c17.bin"; - std::string c18_bin = bin_path + "/layers/c18.bin"; - std::string c19_bin = bin_path + "/layers/c19.bin"; - std::string c20_bin = bin_path + "/layers/c20.bin"; - std::string c21_bin = bin_path + "/layers/c21.bin"; - std::string c23_bin = bin_path + "/layers/c23.bin"; - std::string c24_bin = bin_path + "/layers/c24.bin"; - std::string c25_bin = bin_path + "/layers/c25.bin"; - std::string c26_bin = bin_path + "/layers/c26.bin"; - std::string c27_bin = bin_path + "/layers/c27.bin"; - std::string c28_bin = bin_path + "/layers/c28.bin"; - std::string c29_bin = bin_path + "/layers/c29.bin"; - std::string c30_bin = bin_path + "/layers/c30.bin"; - std::string c31_bin = bin_path + "/layers/c31.bin"; - std::string c32_bin = bin_path + "/layers/c32.bin"; - std::string c33_bin = bin_path + "/layers/c33.bin"; - std::string c34_bin = bin_path + "/layers/c34.bin"; - std::string c35_bin = bin_path + "/layers/c35.bin"; - std::string c36_bin = bin_path + "/layers/c36.bin"; - std::string c37_bin = bin_path + "/layers/c37.bin"; - std::string c38_bin = bin_path + "/layers/c38.bin"; - std::string c39_bin = bin_path + "/layers/c39.bin"; - std::string c40_bin = bin_path + "/layers/c40.bin"; - std::string c41_bin = bin_path + "/layers/c41.bin"; - std::string c42_bin = bin_path + "/layers/c42.bin"; - std::string c43_bin = bin_path + "/layers/c43.bin"; - std::string c44_bin = bin_path + "/layers/c44.bin"; - std::string c45_bin = bin_path + "/layers/c45.bin"; - std::string c46_bin = bin_path + "/layers/c46.bin"; - std::string c47_bin = bin_path + "/layers/c47.bin"; - std::string c48_bin = bin_path + "/layers/c48.bin"; - std::string c49_bin = bin_path + "/layers/c49.bin"; - std::string c50_bin = bin_path + "/layers/c50.bin"; - std::string c51_bin = bin_path + "/layers/c51.bin"; - std::string c52_bin = bin_path + "/layers/c52.bin"; - std::string c53_bin = bin_path + "/layers/c53.bin"; - std::string c54_bin = bin_path + "/layers/c54.bin"; - std::string c55_bin = bin_path + "/layers/c55.bin"; - std::string c56_bin = bin_path + "/layers/c56.bin"; - std::string c57_bin = bin_path + "/layers/c57.bin"; - std::string c58_bin = bin_path + "/layers/c58.bin"; - std::string c59_bin = bin_path + "/layers/c59.bin"; - std::string c60_bin = bin_path + "/layers/c60.bin"; - std::string c61_bin = bin_path + "/layers/c61.bin"; - std::string c62_bin = bin_path + "/layers/c62.bin"; - std::string c63_bin = bin_path + "/layers/c63.bin"; - std::string c65_bin = bin_path + "/layers/c65.bin"; - std::string c66_bin = bin_path + "/layers/c66.bin"; - std::string c67_bin = bin_path + "/layers/c67.bin"; - std::string c68_bin = bin_path + "/layers/c68.bin"; - std::string c69_bin = bin_path + "/layers/c69.bin"; - std::string c70_bin = bin_path + "/layers/c70.bin"; - std::string c71_bin = bin_path + "/layers/c71.bin"; - std::string c72_bin = bin_path + "/layers/c72.bin"; - std::string c74_bin = bin_path + "/layers/c74.bin"; - std::string c75_bin = bin_path + "/layers/c75.bin"; - std::string c76_bin = bin_path + "/layers/c76.bin"; - std::string c77_bin = bin_path + "/layers/c77.bin"; - std::string c78_bin = bin_path + "/layers/c78.bin"; - std::string c80_bin = bin_path + "/layers/c80.bin"; - std::string c81_bin = bin_path + "/layers/c81.bin"; - std::string c82_bin = bin_path + "/layers/c82.bin"; - std::string c83_bin = bin_path + "/layers/c83.bin"; - std::string c85_bin = bin_path + "/layers/c85.bin"; - std::string c86_bin = bin_path + "/layers/c86.bin"; - std::string c87_bin = bin_path + "/layers/c87.bin"; - std::string c89_bin = bin_path + "/layers/c89.bin"; - std::string c90_bin = bin_path + "/layers/c90.bin"; - std::string c91_bin = bin_path + "/layers/c91.bin"; - std::string c92_bin = bin_path + "/layers/c92.bin"; - std::string c93_bin = bin_path + "/layers/c93.bin"; - std::string c94_bin = bin_path + "/layers/c94.bin"; - std::string c96_bin = bin_path + "/layers/c96.bin"; - std::string c97_bin = bin_path + "/layers/c97.bin"; - std::string c98_bin = bin_path + "/layers/c98.bin"; - std::string c99_bin = bin_path + "/layers/c99.bin"; - std::string c100_bin = bin_path + "/layers/c100.bin"; - std::string c101_bin = bin_path + "/layers/c101.bin"; - std::string c102_bin = bin_path + "/layers/c102.bin"; - std::string c103_bin = bin_path + "/layers/c103.bin"; - std::string c104_bin = bin_path + "/layers/c104.bin"; - std::string c105_bin = bin_path + "/layers/c105.bin"; - std::string c106_bin = bin_path + "/layers/c106.bin"; - std::string c107_bin = bin_path + "/layers/c107.bin"; - std::string c108_bin = bin_path + "/layers/c108.bin"; - std::string c109_bin = bin_path + "/layers/c109.bin"; - std::string c110_bin = bin_path + "/layers/c110.bin"; - std::string c111_bin = bin_path + "/layers/c111.bin"; - std::string c112_bin = bin_path + "/layers/c112.bin"; - std::string c113_bin = bin_path + "/layers/c113.bin"; - std::string c114_bin = bin_path + "/layers/c114.bin"; - std::string c115_bin = bin_path + "/layers/c115.bin"; - std::string c116_bin = bin_path + "/layers/c116.bin"; - std::string c117_bin = bin_path + "/layers/c117.bin"; - std::string c119_bin = bin_path + "/layers/c119.bin"; - std::string c120_bin = bin_path + "/layers/c120.bin"; - std::string c121_bin = bin_path + "/layers/c121.bin"; - std::string c122_bin = bin_path + "/layers/c122.bin"; - std::string c123_bin = bin_path + "/layers/c123.bin"; - std::string c124_bin = bin_path + "/layers/c124.bin"; - std::string c125_bin = bin_path + "/layers/c125.bin"; - std::string c126_bin = bin_path + "/layers/c126.bin"; - std::string c127_bin = bin_path + "/layers/c127.bin"; - std::string c128_bin = bin_path + "/layers/c128.bin"; - std::string c130_bin = bin_path + "/layers/c130.bin"; - std::string c131_bin = bin_path + "/layers/c131.bin"; - std::string c132_bin = bin_path + "/layers/c132.bin"; - std::string c133_bin = bin_path + "/layers/c133.bin"; - std::string c134_bin = bin_path + "/layers/c134.bin"; - std::string c135_bin = bin_path + "/layers/c135.bin"; - std::string c136_bin = bin_path + "/layers/c136.bin"; - std::string c137_bin = bin_path + "/layers/c137.bin"; - std::string c138_bin = bin_path + "/layers/c138.bin"; - std::string c141_bin = bin_path + "/layers/c141.bin"; - std::string c142_bin = bin_path + "/layers/c142.bin"; - std::string c143_bin = bin_path + "/layers/c143.bin"; - std::string c144_bin = bin_path + "/layers/c144.bin"; - std::string c145_bin = bin_path + "/layers/c145.bin"; - std::string c146_bin = bin_path + "/layers/c146.bin"; - std::string c147_bin = bin_path + "/layers/c147.bin"; - std::string c148_bin = bin_path + "/layers/c148.bin"; - std::string c149_bin = bin_path + "/layers/c149.bin"; - std::string c150_bin = bin_path + "/layers/c150.bin"; - std::string c151_bin = bin_path + "/layers/c151.bin"; - std::string c152_bin = bin_path + "/layers/c152.bin"; - std::string c153_bin = bin_path + "/layers/c153.bin"; - std::string c154_bin = bin_path + "/layers/c154.bin"; - std::string c155_bin = bin_path + "/layers/c155.bin"; - std::string c156_bin = bin_path + "/layers/c156.bin"; - std::string c157_bin = bin_path + "/layers/c157.bin"; - std::string c158_bin = bin_path + "/layers/c158.bin"; - std::string c159_bin = bin_path + "/layers/c159.bin"; - std::string c160_bin = bin_path + "/layers/c160.bin"; - std::string g139_bin = bin_path + "/layers/g139.bin"; - std::string g150_bin = bin_path + "/layers/g150.bin"; - std::string g161_bin = bin_path + "/layers/g161.bin"; - - - downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); - - tk::dnn::Conv2d c0(&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0(&net, tk::dnn::ACTIVATION_MISH); - - // downsample - tk::dnn::Conv2d c1(&net, 64, 3, 3, 2, 2, 1, 1, c1_bin, true); - tk::dnn::Activation a1(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c2(&net, 64, 1, 1, 1, 1, 0, 0, c2_bin, true); - tk::dnn::Activation a2(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r3_layers[1] = {&a1}; - tk::dnn::Route r3(&net, r3_layers, 1); - - tk::dnn::Conv2d c4(&net, 64, 1, 1, 1, 1, 0, 0, c4_bin, true); - tk::dnn::Activation a4(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c5(&net, 32, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c6(&net, 64, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s7(&net, &a4); - - tk::dnn::Conv2d c8(&net, 64, 1, 1, 1, 1, 0, 0, c8_bin, true); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r9_layers[2] = {&a8, &a2}; - tk::dnn::Route r9(&net, r9_layers, 2); - - tk::dnn::Conv2d c10(&net, 64, 1, 1, 1, 1, 0, 0, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_MISH); - - // downsample - tk::dnn::Conv2d c11(&net, 128, 3, 3, 2, 2, 1, 1, c11_bin, true); - tk::dnn::Activation a11(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c12(&net, 64, 1, 1, 1, 1, 0, 0, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r13_layers[1] = {&a11}; - tk::dnn::Route r13(&net, r13_layers, 1); - - tk::dnn::Conv2d c14(&net, 64, 1, 1, 1, 1, 0, 0, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c15(&net, 64, 1, 1, 1, 1, 0, 0, c15_bin, true); - tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c16(&net, 64, 3, 3, 1, 1, 1, 1, c16_bin, true); - tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s17(&net, &a14); - - tk::dnn::Conv2d c18(&net, 64, 1, 1, 1, 1, 0, 0, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c19(&net, 64, 3, 3, 1, 1, 1, 1, c19_bin, true); - tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s20(&net, &s17); - - tk::dnn::Conv2d c21(&net, 64, 1, 1, 1, 1, 0, 0, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r22_layers[2] = {&a21, &a12}; - tk::dnn::Route r22(&net, r22_layers, 2); - - tk::dnn::Conv2d c23(&net, 128, 1, 1, 1, 1, 0, 0, c23_bin, true); - tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c24(&net, 256, 3, 3, 2, 2, 1, 1, c24_bin, true); - tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c25(&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true); - tk::dnn::Activation a25(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r26_layers[1] = {&a24}; - tk::dnn::Route r26(&net, r26_layers, 1); - - tk::dnn::Conv2d c27(&net, 128, 1, 1, 1, 1, 0, 0, c27_bin, true); - tk::dnn::Activation a27(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c28(&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true); - tk::dnn::Activation a28(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c29(&net, 128, 3, 3, 1, 1, 1, 1, c29_bin, true); - tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s30(&net, &a27); - - tk::dnn::Conv2d c31(&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true); - tk::dnn::Activation a31(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c32(&net, 128, 3, 3, 1, 1, 1, 1, c32_bin, true); - tk::dnn::Activation a32(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s33(&net, &s30); - - tk::dnn::Conv2d c34(&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true); - tk::dnn::Activation a34(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c35(&net, 128, 3, 3, 1, 1, 1, 1, c35_bin, true); - tk::dnn::Activation a35(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s36(&net, &s33); - - tk::dnn::Conv2d c37(&net, 128, 1, 1, 1, 1, 0, 0, c37_bin, true); - tk::dnn::Activation a37(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c38(&net, 128, 3, 3, 1, 1, 1, 1, c38_bin, true); - tk::dnn::Activation a38(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s39(&net, &s36); - - tk::dnn::Conv2d c40(&net, 128, 1, 1, 1, 1, 0, 0, c40_bin, true); - tk::dnn::Activation a40(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c41(&net, 128, 3, 3, 1, 1, 1, 1, c41_bin, true); - tk::dnn::Activation a41(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s42(&net, &s39); - - tk::dnn::Conv2d c43(&net, 128, 1, 1, 1, 1, 0, 0, c43_bin, true); - tk::dnn::Activation a43(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c44(&net, 128, 3, 3, 1, 1, 1, 1, c44_bin, true); - tk::dnn::Activation a44(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s45(&net, &s42); - - tk::dnn::Conv2d c46(&net, 128, 1, 1, 1, 1, 0, 0, c46_bin, true); - tk::dnn::Activation a46(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c47(&net, 128, 3, 3, 1, 1, 1, 1, c47_bin, true); - tk::dnn::Activation a47(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s48(&net, &s45); - - tk::dnn::Conv2d c49(&net, 128, 1, 1, 1, 1, 0, 0, c49_bin, true); - tk::dnn::Activation a49(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c50(&net, 128, 3, 3, 1, 1, 1, 1, c50_bin, true); - tk::dnn::Activation a50(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s51(&net, &s48); - - tk::dnn::Conv2d c52(&net, 128, 1, 1, 1, 1, 0, 0, c52_bin, true); - tk::dnn::Activation a52(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r53_layers[2] = {&a52, &a25}; - tk::dnn::Route r53(&net, r53_layers, 2); - - tk::dnn::Conv2d c54(&net, 256, 1, 1, 1, 1, 0, 0, c54_bin, true); - tk::dnn::Activation a54(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c55(&net, 512, 3, 3, 2, 2, 1, 1, c55_bin, true); - tk::dnn::Activation a55(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c56(&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true); - tk::dnn::Activation a56(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r57_layers[1] = {&a55}; - tk::dnn::Route r57(&net, r57_layers, 1); - - tk::dnn::Conv2d c58(&net, 256, 1, 1, 1, 1, 0, 0, c58_bin, true); - tk::dnn::Activation a58(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c59(&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true); - tk::dnn::Activation a59(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c60(&net, 256, 3, 3, 1, 1, 1, 1, c60_bin, true); - tk::dnn::Activation a60(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s61(&net, &a58); - - tk::dnn::Conv2d c62(&net, 256, 1, 1, 1, 1, 0, 0, c62_bin, true); - tk::dnn::Activation a62(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c63(&net, 256, 3, 3, 1, 1, 1, 1, c63_bin, true); - tk::dnn::Activation a63(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s64(&net, &s61); - - tk::dnn::Conv2d c65(&net, 256, 1, 1, 1, 1, 0, 0, c65_bin, true); - tk::dnn::Activation a65(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c66(&net, 256, 3, 3, 1, 1, 1, 1, c66_bin, true); - tk::dnn::Activation a66(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s67(&net, &s64); - - tk::dnn::Conv2d c68(&net, 256, 1, 1, 1, 1, 0, 0, c68_bin, true); - tk::dnn::Activation a68(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c69(&net, 256, 3, 3, 1, 1, 1, 1, c69_bin, true); - tk::dnn::Activation a69(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s70(&net, &s67); - - tk::dnn::Conv2d c71(&net, 256, 1, 1, 1, 1, 0, 0, c71_bin, true); - tk::dnn::Activation a71(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c72(&net, 256, 3, 3, 1, 1, 1, 1, c72_bin, true); - tk::dnn::Activation a72(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s73(&net, &s70); - - tk::dnn::Conv2d c74(&net, 256, 1, 1, 1, 1, 0, 0, c74_bin, true); - tk::dnn::Activation a74(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c75(&net, 256, 3, 3, 1, 1, 1, 1, c75_bin, true); - tk::dnn::Activation a75(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s76(&net, &s73); - - tk::dnn::Conv2d c77(&net, 256, 1, 1, 1, 1, 0, 0, c77_bin, true); - tk::dnn::Activation a77(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c78(&net, 256, 3, 3, 1, 1, 1, 1, c78_bin, true); - tk::dnn::Activation a78(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s79(&net, &s76); - - tk::dnn::Conv2d c80(&net, 256, 1, 1, 1, 1, 0, 0, c80_bin, true); - tk::dnn::Activation a80(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c81(&net, 256, 3, 3, 1, 1, 1, 1, c81_bin, true); - tk::dnn::Activation a81(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s82(&net, &s79); - - tk::dnn::Conv2d c83(&net, 256, 1, 1, 1, 1, 0, 0, c83_bin, true); - tk::dnn::Activation a83(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r84_layers[2] = {&a83, &a56}; - tk::dnn::Route r84(&net, r84_layers, 2); - - tk::dnn::Conv2d c85(&net, 512, 1, 1, 1, 1, 0, 0, c85_bin, true); - tk::dnn::Activation a85(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c86(&net, 1024, 3, 3, 2, 2, 1, 1, c86_bin, true); - tk::dnn::Activation a86(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c87(&net, 512, 1, 1, 1, 1, 0, 0, c87_bin, true); - tk::dnn::Activation a87(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r88_layers[1] = {&a86}; - tk::dnn::Route r88(&net, r88_layers, 1); - - tk::dnn::Conv2d c89(&net, 512, 1, 1, 1, 1, 0, 0, c89_bin, true); - tk::dnn::Activation a89(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c90(&net, 512, 1, 1, 1, 1, 0, 0, c90_bin, true); - tk::dnn::Activation a90(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c91(&net, 512, 3, 3, 1, 1, 1, 1, c91_bin, true); - tk::dnn::Activation a91(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s92(&net, &a89); - - tk::dnn::Conv2d c93(&net, 512, 1, 1, 1, 1, 0, 0, c93_bin, true); - tk::dnn::Activation a93(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c94(&net, 512, 3, 3, 1, 1, 1, 1, c94_bin, true); - tk::dnn::Activation a94(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s95(&net, &s92); - - tk::dnn::Conv2d c96(&net, 512, 1, 1, 1, 1, 0, 0, c96_bin, true); - tk::dnn::Activation a96(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c97(&net, 512, 3, 3, 1, 1, 1, 1, c97_bin, true); - tk::dnn::Activation a97(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s98(&net, &s95); - - tk::dnn::Conv2d c99(&net, 512, 1, 1, 1, 1, 0, 0, c99_bin, true); - tk::dnn::Activation a99(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c100(&net, 512, 3, 3, 1, 1, 1, 1, c100_bin, true); - tk::dnn::Activation a100(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s101(&net, &s98); - - tk::dnn::Conv2d c102(&net, 512, 1, 1, 1, 1, 0, 0, c102_bin, true); - tk::dnn::Activation a102(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r103_layers[2] = {&a102, &a87}; - tk::dnn::Route r103(&net, r103_layers, 2); - - tk::dnn::Conv2d c104(&net, 1024, 1, 1, 1, 1, 0, 0, c104_bin, true); - tk::dnn::Activation a104(&net, tk::dnn::ACTIVATION_MISH); - - - //################ - tk::dnn::Conv2d c105(&net, 512, 1, 1, 1, 1, 0, 0, c105_bin, true); - tk::dnn::Activation a105(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c106(&net, 1024, 3, 3, 1, 1, 1, 1, c106_bin, true); - tk::dnn::Activation a106(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c107(&net, 512, 1, 1, 1, 1, 0, 0, c107_bin, true); - tk::dnn::Activation a107(&net, tk::dnn::ACTIVATION_LEAKY); - - //SPP - tk::dnn::Pooling p108(&net, 5, 5, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r109_layers[1] = {&a107}; - tk::dnn::Route r109(&net, r109_layers, 1); - - tk::dnn::Pooling p110(&net, 9, 9, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r111_layers[1] = {&a107}; - tk::dnn::Route r111(&net, r111_layers, 1); - - tk::dnn::Pooling p112(&net, 13, 13, 1, 1, 12, 12, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r113_layers[4] = {&p112, &p110, &p108, &a107}; - tk::dnn::Route r113(&net, r113_layers, 4); - //END SPP - - tk::dnn::Conv2d c114(&net, 512, 1, 1, 1, 1, 0, 0, c114_bin, true); - tk::dnn::Activation a114(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c115(&net, 1024, 3, 3, 1, 1, 1, 1, c115_bin, true); - tk::dnn::Activation a115(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c116(&net, 512, 1, 1, 1, 1, 0, 0, c116_bin, true); - tk::dnn::Activation a116(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c117(&net, 256, 1, 1, 1, 1, 0, 0, c117_bin, true); - tk::dnn::Activation a117(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u118(&net, 2); - tk::dnn::Layer *r119_layers[1] = {&a85}; - tk::dnn::Route r119(&net, r119_layers, 1); - tk::dnn::Conv2d c120(&net, 256, 1, 1, 1, 1, 0, 0, c120_bin, true); - tk::dnn::Activation a120(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r121_layers[2] = {&a120,&u118}; - tk::dnn::Route r121(&net, r121_layers, 2); - - tk::dnn::Conv2d c122(&net, 256, 1, 1, 1, 1, 0, 0, c122_bin, true); - tk::dnn::Activation a122(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c123(&net, 512, 3, 3, 1, 1, 1, 1, c123_bin, true); - tk::dnn::Activation a123(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c124(&net, 256, 1, 1, 1, 1, 0, 0, c124_bin, true); - tk::dnn::Activation a124(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c125(&net, 512, 3, 3, 1, 1, 1, 1, c125_bin, true); - tk::dnn::Activation a125(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c126(&net, 256, 1, 1, 1, 1, 0, 0, c126_bin, true); - tk::dnn::Activation a126(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c127(&net, 128, 1, 1, 1, 1, 0, 0, c127_bin, true); - tk::dnn::Activation a127(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u128(&net, 2); - tk::dnn::Layer *r129_layers[1] = {&a54}; - tk::dnn::Route r129(&net, r129_layers, 1); - tk::dnn::Conv2d c130(&net, 128, 1, 1, 1, 1, 0, 0, c130_bin, true); - tk::dnn::Activation a130(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r131_layers[2] = {&a130,&u128}; - tk::dnn::Route r131(&net, r131_layers, 2); - - - tk::dnn::Conv2d c132(&net, 128, 1, 1, 1, 1, 0, 0, c132_bin, true); - tk::dnn::Activation a132(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c133(&net, 256, 3, 3, 1, 1, 1, 1, c133_bin, true); - tk::dnn::Activation a133(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c134(&net, 128, 1, 1, 1, 1, 0, 0, c134_bin, true); - tk::dnn::Activation a134(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c135(&net, 256, 3, 3, 1, 1, 1, 1, c135_bin, true); - tk::dnn::Activation a135(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c136(&net, 128, 1, 1, 1, 1, 0, 0, c136_bin, true); - tk::dnn::Activation a136(&net, tk::dnn::ACTIVATION_LEAKY); - - - tk::dnn::Conv2d c137(&net, 256, 3, 3, 1, 1, 1, 1, c137_bin, true); - tk::dnn::Activation a137(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c138(&net, 255, 1, 1, 1, 1, 0, 0, c138_bin, false); - tk::dnn::Yolo yolo139(&net, classes, 3, g139_bin, 3, 1.2); - - tk::dnn::Layer *r140_layers[1] = {&a136}; - tk::dnn::Route r140(&net, r140_layers, 1); - tk::dnn::Conv2d c141(&net, 256, 3, 3, 2, 2, 1, 1, c141_bin, true); - tk::dnn::Activation a141(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r142_layers[2] = {&a141,&a126}; - tk::dnn::Route r142(&net, r142_layers, 2); - - tk::dnn::Conv2d c143(&net, 256, 1, 1, 1, 1, 0, 0, c143_bin, true); - tk::dnn::Activation a143(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c144(&net, 512, 3, 3, 1, 1, 1, 1, c144_bin, true); - tk::dnn::Activation a144(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c145(&net, 256, 1, 1, 1, 1, 0, 0, c145_bin, true); - tk::dnn::Activation a145(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c146(&net, 512, 3, 3, 1, 1, 1, 1, c146_bin, true); - tk::dnn::Activation a146(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c147(&net, 256, 1, 1, 1, 1, 0, 0, c147_bin, true); - tk::dnn::Activation a147(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c148(&net, 512, 3, 3, 1, 1, 1, 1, c148_bin, true); - tk::dnn::Activation a148(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c149(&net, 255, 1, 1, 1, 1, 0, 0, c149_bin, false); - tk::dnn::Yolo yolo150(&net, classes, 3, g150_bin, 3, 1.1); - - tk::dnn::Layer *r151_layers[1] = {&a147}; - tk::dnn::Route r151(&net, r151_layers, 1); - tk::dnn::Conv2d c152(&net, 512, 3, 3, 2, 2, 1, 1, c152_bin, true); - tk::dnn::Activation a152(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r153_layers[2] = {&a152,&a116}; - tk::dnn::Route r153(&net, r153_layers, 2); - - tk::dnn::Conv2d c154(&net, 512, 1, 1, 1, 1, 0, 0, c154_bin, true); - tk::dnn::Activation a154(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c155(&net, 1024, 3, 3, 1, 1, 1, 1, c155_bin, true); - tk::dnn::Activation a155(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c156(&net, 512, 1, 1, 1, 1, 0, 0, c156_bin, true); - tk::dnn::Activation a156(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c157(&net, 1024, 3, 3, 1, 1, 1, 1, c157_bin, true); - tk::dnn::Activation a157(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c158(&net, 512, 1, 1, 1, 1, 0, 0, c158_bin, true); - tk::dnn::Activation a158(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c159(&net, 1024, 3, 3, 1, 1, 1, 1, c159_bin, true); - tk::dnn::Activation a159(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c160(&net, 255, 1, 1, 1, 1, 0, 0, c160_bin, false); - tk::dnn::Yolo yolo161(&net, classes, 3, g161_bin, 3, 1.05); - - - - - - - yolo[0] = &yolo139; - yolo[1] = &yolo150; - yolo[2] = &yolo161; - - // fill classes names - for (int i = 0; i < 3; i++) - { - yolo[i]->classesNames = {"person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"}; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - // //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo4_320")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for (int i = 0; i < 3; i++) - out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); - { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - for (int i = 0; i < 3; i++) - cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for (int i = 0; i < 3; i++) - yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for (int j = 0; j < ndets; j++) - { - tk::dnn::Yolo::box b = dets[j].bbox; - int x0 = (b.x - b.w / 2.); - int x1 = (b.x + b.w / 2.); - int y0 = (b.y - b.h / 2.); - int y1 = (b.y + b.h / 2.); - - int cl = 0; - for (int c = 0; c < classes; ++c) - { - float prob = dets[j].prob[c]; - if (prob > 0) - cl = c; - } - std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; - } - TIMER_STOP - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); - { - dim2.print(); - TIMER_START - netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - for (int i = 0; i < 3; i++) - rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; - - int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; - for (int i = 0; i < 3; i++) - { - printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); - dnnType *out, *out_h; - int odim = out_dim[i].tot(); - readBinaryFile(output_bins[i], odim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - } - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo4/yolo4_512.cpp b/tests/yolo4/yolo4_512.cpp deleted file mode 100644 index 963df75..0000000 --- a/tests/yolo4/yolo4_512.cpp +++ /dev/null @@ -1,666 +0,0 @@ -#include -#include -#include "tkdnn.h" - -int main() -{ - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 512, 512, 1); - tk::dnn::Network net(dim); - - // create yolo4_512 model - std::string bin_path = "yolo4_512"; - int classes = 80; - tk::dnn::Yolo *yolo[3]; - - std::string input_bin = bin_path + "/layers/input.bin"; - - std::vector output_bins = { - bin_path + "/debug/layer139_out.bin", - bin_path + "/debug/layer150_out.bin", - bin_path + "/debug/layer161_out.bin"}; - std::string c0_bin = bin_path + "/layers/c0.bin"; - std::string c1_bin = bin_path + "/layers/c1.bin"; - std::string c2_bin = bin_path + "/layers/c2.bin"; - std::string c3_bin = bin_path + "/layers/c3.bin"; - std::string c4_bin = bin_path + "/layers/c4.bin"; - std::string c5_bin = bin_path + "/layers/c5.bin"; - std::string c6_bin = bin_path + "/layers/c6.bin"; - std::string c7_bin = bin_path + "/layers/c7.bin"; - std::string c8_bin = bin_path + "/layers/c8.bin"; - std::string c10_bin = bin_path + "/layers/c10.bin"; - std::string c11_bin = bin_path + "/layers/c11.bin"; - std::string c12_bin = bin_path + "/layers/c12.bin"; - std::string c13_bin = bin_path + "/layers/c13.bin"; - std::string c14_bin = bin_path + "/layers/c14.bin"; - std::string c15_bin = bin_path + "/layers/c15.bin"; - std::string c16_bin = bin_path + "/layers/c16.bin"; - std::string c17_bin = bin_path + "/layers/c17.bin"; - std::string c18_bin = bin_path + "/layers/c18.bin"; - std::string c19_bin = bin_path + "/layers/c19.bin"; - std::string c20_bin = bin_path + "/layers/c20.bin"; - std::string c21_bin = bin_path + "/layers/c21.bin"; - std::string c23_bin = bin_path + "/layers/c23.bin"; - std::string c24_bin = bin_path + "/layers/c24.bin"; - std::string c25_bin = bin_path + "/layers/c25.bin"; - std::string c26_bin = bin_path + "/layers/c26.bin"; - std::string c27_bin = bin_path + "/layers/c27.bin"; - std::string c28_bin = bin_path + "/layers/c28.bin"; - std::string c29_bin = bin_path + "/layers/c29.bin"; - std::string c30_bin = bin_path + "/layers/c30.bin"; - std::string c31_bin = bin_path + "/layers/c31.bin"; - std::string c32_bin = bin_path + "/layers/c32.bin"; - std::string c33_bin = bin_path + "/layers/c33.bin"; - std::string c34_bin = bin_path + "/layers/c34.bin"; - std::string c35_bin = bin_path + "/layers/c35.bin"; - std::string c36_bin = bin_path + "/layers/c36.bin"; - std::string c37_bin = bin_path + "/layers/c37.bin"; - std::string c38_bin = bin_path + "/layers/c38.bin"; - std::string c39_bin = bin_path + "/layers/c39.bin"; - std::string c40_bin = bin_path + "/layers/c40.bin"; - std::string c41_bin = bin_path + "/layers/c41.bin"; - std::string c42_bin = bin_path + "/layers/c42.bin"; - std::string c43_bin = bin_path + "/layers/c43.bin"; - std::string c44_bin = bin_path + "/layers/c44.bin"; - std::string c45_bin = bin_path + "/layers/c45.bin"; - std::string c46_bin = bin_path + "/layers/c46.bin"; - std::string c47_bin = bin_path + "/layers/c47.bin"; - std::string c48_bin = bin_path + "/layers/c48.bin"; - std::string c49_bin = bin_path + "/layers/c49.bin"; - std::string c50_bin = bin_path + "/layers/c50.bin"; - std::string c51_bin = bin_path + "/layers/c51.bin"; - std::string c52_bin = bin_path + "/layers/c52.bin"; - std::string c53_bin = bin_path + "/layers/c53.bin"; - std::string c54_bin = bin_path + "/layers/c54.bin"; - std::string c55_bin = bin_path + "/layers/c55.bin"; - std::string c56_bin = bin_path + "/layers/c56.bin"; - std::string c57_bin = bin_path + "/layers/c57.bin"; - std::string c58_bin = bin_path + "/layers/c58.bin"; - std::string c59_bin = bin_path + "/layers/c59.bin"; - std::string c60_bin = bin_path + "/layers/c60.bin"; - std::string c61_bin = bin_path + "/layers/c61.bin"; - std::string c62_bin = bin_path + "/layers/c62.bin"; - std::string c63_bin = bin_path + "/layers/c63.bin"; - std::string c65_bin = bin_path + "/layers/c65.bin"; - std::string c66_bin = bin_path + "/layers/c66.bin"; - std::string c67_bin = bin_path + "/layers/c67.bin"; - std::string c68_bin = bin_path + "/layers/c68.bin"; - std::string c69_bin = bin_path + "/layers/c69.bin"; - std::string c70_bin = bin_path + "/layers/c70.bin"; - std::string c71_bin = bin_path + "/layers/c71.bin"; - std::string c72_bin = bin_path + "/layers/c72.bin"; - std::string c74_bin = bin_path + "/layers/c74.bin"; - std::string c75_bin = bin_path + "/layers/c75.bin"; - std::string c76_bin = bin_path + "/layers/c76.bin"; - std::string c77_bin = bin_path + "/layers/c77.bin"; - std::string c78_bin = bin_path + "/layers/c78.bin"; - std::string c80_bin = bin_path + "/layers/c80.bin"; - std::string c81_bin = bin_path + "/layers/c81.bin"; - std::string c82_bin = bin_path + "/layers/c82.bin"; - std::string c83_bin = bin_path + "/layers/c83.bin"; - std::string c85_bin = bin_path + "/layers/c85.bin"; - std::string c86_bin = bin_path + "/layers/c86.bin"; - std::string c87_bin = bin_path + "/layers/c87.bin"; - std::string c89_bin = bin_path + "/layers/c89.bin"; - std::string c90_bin = bin_path + "/layers/c90.bin"; - std::string c91_bin = bin_path + "/layers/c91.bin"; - std::string c92_bin = bin_path + "/layers/c92.bin"; - std::string c93_bin = bin_path + "/layers/c93.bin"; - std::string c94_bin = bin_path + "/layers/c94.bin"; - std::string c96_bin = bin_path + "/layers/c96.bin"; - std::string c97_bin = bin_path + "/layers/c97.bin"; - std::string c98_bin = bin_path + "/layers/c98.bin"; - std::string c99_bin = bin_path + "/layers/c99.bin"; - std::string c100_bin = bin_path + "/layers/c100.bin"; - std::string c101_bin = bin_path + "/layers/c101.bin"; - std::string c102_bin = bin_path + "/layers/c102.bin"; - std::string c103_bin = bin_path + "/layers/c103.bin"; - std::string c104_bin = bin_path + "/layers/c104.bin"; - std::string c105_bin = bin_path + "/layers/c105.bin"; - std::string c106_bin = bin_path + "/layers/c106.bin"; - std::string c107_bin = bin_path + "/layers/c107.bin"; - std::string c108_bin = bin_path + "/layers/c108.bin"; - std::string c109_bin = bin_path + "/layers/c109.bin"; - std::string c110_bin = bin_path + "/layers/c110.bin"; - std::string c111_bin = bin_path + "/layers/c111.bin"; - std::string c112_bin = bin_path + "/layers/c112.bin"; - std::string c113_bin = bin_path + "/layers/c113.bin"; - std::string c114_bin = bin_path + "/layers/c114.bin"; - std::string c115_bin = bin_path + "/layers/c115.bin"; - std::string c116_bin = bin_path + "/layers/c116.bin"; - std::string c117_bin = bin_path + "/layers/c117.bin"; - std::string c119_bin = bin_path + "/layers/c119.bin"; - std::string c120_bin = bin_path + "/layers/c120.bin"; - std::string c121_bin = bin_path + "/layers/c121.bin"; - std::string c122_bin = bin_path + "/layers/c122.bin"; - std::string c123_bin = bin_path + "/layers/c123.bin"; - std::string c124_bin = bin_path + "/layers/c124.bin"; - std::string c125_bin = bin_path + "/layers/c125.bin"; - std::string c126_bin = bin_path + "/layers/c126.bin"; - std::string c127_bin = bin_path + "/layers/c127.bin"; - std::string c128_bin = bin_path + "/layers/c128.bin"; - std::string c130_bin = bin_path + "/layers/c130.bin"; - std::string c131_bin = bin_path + "/layers/c131.bin"; - std::string c132_bin = bin_path + "/layers/c132.bin"; - std::string c133_bin = bin_path + "/layers/c133.bin"; - std::string c134_bin = bin_path + "/layers/c134.bin"; - std::string c135_bin = bin_path + "/layers/c135.bin"; - std::string c136_bin = bin_path + "/layers/c136.bin"; - std::string c137_bin = bin_path + "/layers/c137.bin"; - std::string c138_bin = bin_path + "/layers/c138.bin"; - std::string c141_bin = bin_path + "/layers/c141.bin"; - std::string c142_bin = bin_path + "/layers/c142.bin"; - std::string c143_bin = bin_path + "/layers/c143.bin"; - std::string c144_bin = bin_path + "/layers/c144.bin"; - std::string c145_bin = bin_path + "/layers/c145.bin"; - std::string c146_bin = bin_path + "/layers/c146.bin"; - std::string c147_bin = bin_path + "/layers/c147.bin"; - std::string c148_bin = bin_path + "/layers/c148.bin"; - std::string c149_bin = bin_path + "/layers/c149.bin"; - std::string c150_bin = bin_path + "/layers/c150.bin"; - std::string c151_bin = bin_path + "/layers/c151.bin"; - std::string c152_bin = bin_path + "/layers/c152.bin"; - std::string c153_bin = bin_path + "/layers/c153.bin"; - std::string c154_bin = bin_path + "/layers/c154.bin"; - std::string c155_bin = bin_path + "/layers/c155.bin"; - std::string c156_bin = bin_path + "/layers/c156.bin"; - std::string c157_bin = bin_path + "/layers/c157.bin"; - std::string c158_bin = bin_path + "/layers/c158.bin"; - std::string c159_bin = bin_path + "/layers/c159.bin"; - std::string c160_bin = bin_path + "/layers/c160.bin"; - std::string g139_bin = bin_path + "/layers/g139.bin"; - std::string g150_bin = bin_path + "/layers/g150.bin"; - std::string g161_bin = bin_path + "/layers/g161.bin"; - - - downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); - - tk::dnn::Conv2d c0(&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0(&net, tk::dnn::ACTIVATION_MISH); - - // downsample - tk::dnn::Conv2d c1(&net, 64, 3, 3, 2, 2, 1, 1, c1_bin, true); - tk::dnn::Activation a1(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c2(&net, 64, 1, 1, 1, 1, 0, 0, c2_bin, true); - tk::dnn::Activation a2(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r3_layers[1] = {&a1}; - tk::dnn::Route r3(&net, r3_layers, 1); - - tk::dnn::Conv2d c4(&net, 64, 1, 1, 1, 1, 0, 0, c4_bin, true); - tk::dnn::Activation a4(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c5(&net, 32, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c6(&net, 64, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s7(&net, &a4); - - tk::dnn::Conv2d c8(&net, 64, 1, 1, 1, 1, 0, 0, c8_bin, true); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r9_layers[2] = {&a8, &a2}; - tk::dnn::Route r9(&net, r9_layers, 2); - - tk::dnn::Conv2d c10(&net, 64, 1, 1, 1, 1, 0, 0, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_MISH); - - // downsample - tk::dnn::Conv2d c11(&net, 128, 3, 3, 2, 2, 1, 1, c11_bin, true); - tk::dnn::Activation a11(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c12(&net, 64, 1, 1, 1, 1, 0, 0, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r13_layers[1] = {&a11}; - tk::dnn::Route r13(&net, r13_layers, 1); - - tk::dnn::Conv2d c14(&net, 64, 1, 1, 1, 1, 0, 0, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c15(&net, 64, 1, 1, 1, 1, 0, 0, c15_bin, true); - tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c16(&net, 64, 3, 3, 1, 1, 1, 1, c16_bin, true); - tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s17(&net, &a14); - - tk::dnn::Conv2d c18(&net, 64, 1, 1, 1, 1, 0, 0, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c19(&net, 64, 3, 3, 1, 1, 1, 1, c19_bin, true); - tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s20(&net, &s17); - - tk::dnn::Conv2d c21(&net, 64, 1, 1, 1, 1, 0, 0, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r22_layers[2] = {&a21, &a12}; - tk::dnn::Route r22(&net, r22_layers, 2); - - tk::dnn::Conv2d c23(&net, 128, 1, 1, 1, 1, 0, 0, c23_bin, true); - tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c24(&net, 256, 3, 3, 2, 2, 1, 1, c24_bin, true); - tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c25(&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true); - tk::dnn::Activation a25(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r26_layers[1] = {&a24}; - tk::dnn::Route r26(&net, r26_layers, 1); - - tk::dnn::Conv2d c27(&net, 128, 1, 1, 1, 1, 0, 0, c27_bin, true); - tk::dnn::Activation a27(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c28(&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true); - tk::dnn::Activation a28(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c29(&net, 128, 3, 3, 1, 1, 1, 1, c29_bin, true); - tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s30(&net, &a27); - - tk::dnn::Conv2d c31(&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true); - tk::dnn::Activation a31(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c32(&net, 128, 3, 3, 1, 1, 1, 1, c32_bin, true); - tk::dnn::Activation a32(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s33(&net, &s30); - - tk::dnn::Conv2d c34(&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true); - tk::dnn::Activation a34(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c35(&net, 128, 3, 3, 1, 1, 1, 1, c35_bin, true); - tk::dnn::Activation a35(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s36(&net, &s33); - - tk::dnn::Conv2d c37(&net, 128, 1, 1, 1, 1, 0, 0, c37_bin, true); - tk::dnn::Activation a37(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c38(&net, 128, 3, 3, 1, 1, 1, 1, c38_bin, true); - tk::dnn::Activation a38(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s39(&net, &s36); - - tk::dnn::Conv2d c40(&net, 128, 1, 1, 1, 1, 0, 0, c40_bin, true); - tk::dnn::Activation a40(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c41(&net, 128, 3, 3, 1, 1, 1, 1, c41_bin, true); - tk::dnn::Activation a41(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s42(&net, &s39); - - tk::dnn::Conv2d c43(&net, 128, 1, 1, 1, 1, 0, 0, c43_bin, true); - tk::dnn::Activation a43(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c44(&net, 128, 3, 3, 1, 1, 1, 1, c44_bin, true); - tk::dnn::Activation a44(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s45(&net, &s42); - - tk::dnn::Conv2d c46(&net, 128, 1, 1, 1, 1, 0, 0, c46_bin, true); - tk::dnn::Activation a46(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c47(&net, 128, 3, 3, 1, 1, 1, 1, c47_bin, true); - tk::dnn::Activation a47(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s48(&net, &s45); - - tk::dnn::Conv2d c49(&net, 128, 1, 1, 1, 1, 0, 0, c49_bin, true); - tk::dnn::Activation a49(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c50(&net, 128, 3, 3, 1, 1, 1, 1, c50_bin, true); - tk::dnn::Activation a50(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s51(&net, &s48); - - tk::dnn::Conv2d c52(&net, 128, 1, 1, 1, 1, 0, 0, c52_bin, true); - tk::dnn::Activation a52(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r53_layers[2] = {&a52, &a25}; - tk::dnn::Route r53(&net, r53_layers, 2); - - tk::dnn::Conv2d c54(&net, 256, 1, 1, 1, 1, 0, 0, c54_bin, true); - tk::dnn::Activation a54(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c55(&net, 512, 3, 3, 2, 2, 1, 1, c55_bin, true); - tk::dnn::Activation a55(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c56(&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true); - tk::dnn::Activation a56(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r57_layers[1] = {&a55}; - tk::dnn::Route r57(&net, r57_layers, 1); - - tk::dnn::Conv2d c58(&net, 256, 1, 1, 1, 1, 0, 0, c58_bin, true); - tk::dnn::Activation a58(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c59(&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true); - tk::dnn::Activation a59(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c60(&net, 256, 3, 3, 1, 1, 1, 1, c60_bin, true); - tk::dnn::Activation a60(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s61(&net, &a58); - - tk::dnn::Conv2d c62(&net, 256, 1, 1, 1, 1, 0, 0, c62_bin, true); - tk::dnn::Activation a62(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c63(&net, 256, 3, 3, 1, 1, 1, 1, c63_bin, true); - tk::dnn::Activation a63(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s64(&net, &s61); - - tk::dnn::Conv2d c65(&net, 256, 1, 1, 1, 1, 0, 0, c65_bin, true); - tk::dnn::Activation a65(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c66(&net, 256, 3, 3, 1, 1, 1, 1, c66_bin, true); - tk::dnn::Activation a66(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s67(&net, &s64); - - tk::dnn::Conv2d c68(&net, 256, 1, 1, 1, 1, 0, 0, c68_bin, true); - tk::dnn::Activation a68(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c69(&net, 256, 3, 3, 1, 1, 1, 1, c69_bin, true); - tk::dnn::Activation a69(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s70(&net, &s67); - - tk::dnn::Conv2d c71(&net, 256, 1, 1, 1, 1, 0, 0, c71_bin, true); - tk::dnn::Activation a71(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c72(&net, 256, 3, 3, 1, 1, 1, 1, c72_bin, true); - tk::dnn::Activation a72(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s73(&net, &s70); - - tk::dnn::Conv2d c74(&net, 256, 1, 1, 1, 1, 0, 0, c74_bin, true); - tk::dnn::Activation a74(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c75(&net, 256, 3, 3, 1, 1, 1, 1, c75_bin, true); - tk::dnn::Activation a75(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s76(&net, &s73); - - tk::dnn::Conv2d c77(&net, 256, 1, 1, 1, 1, 0, 0, c77_bin, true); - tk::dnn::Activation a77(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c78(&net, 256, 3, 3, 1, 1, 1, 1, c78_bin, true); - tk::dnn::Activation a78(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s79(&net, &s76); - - tk::dnn::Conv2d c80(&net, 256, 1, 1, 1, 1, 0, 0, c80_bin, true); - tk::dnn::Activation a80(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c81(&net, 256, 3, 3, 1, 1, 1, 1, c81_bin, true); - tk::dnn::Activation a81(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s82(&net, &s79); - - tk::dnn::Conv2d c83(&net, 256, 1, 1, 1, 1, 0, 0, c83_bin, true); - tk::dnn::Activation a83(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r84_layers[2] = {&a83, &a56}; - tk::dnn::Route r84(&net, r84_layers, 2); - - tk::dnn::Conv2d c85(&net, 512, 1, 1, 1, 1, 0, 0, c85_bin, true); - tk::dnn::Activation a85(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c86(&net, 1024, 3, 3, 2, 2, 1, 1, c86_bin, true); - tk::dnn::Activation a86(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c87(&net, 512, 1, 1, 1, 1, 0, 0, c87_bin, true); - tk::dnn::Activation a87(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r88_layers[1] = {&a86}; - tk::dnn::Route r88(&net, r88_layers, 1); - - tk::dnn::Conv2d c89(&net, 512, 1, 1, 1, 1, 0, 0, c89_bin, true); - tk::dnn::Activation a89(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c90(&net, 512, 1, 1, 1, 1, 0, 0, c90_bin, true); - tk::dnn::Activation a90(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c91(&net, 512, 3, 3, 1, 1, 1, 1, c91_bin, true); - tk::dnn::Activation a91(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s92(&net, &a89); - - tk::dnn::Conv2d c93(&net, 512, 1, 1, 1, 1, 0, 0, c93_bin, true); - tk::dnn::Activation a93(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c94(&net, 512, 3, 3, 1, 1, 1, 1, c94_bin, true); - tk::dnn::Activation a94(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s95(&net, &s92); - - tk::dnn::Conv2d c96(&net, 512, 1, 1, 1, 1, 0, 0, c96_bin, true); - tk::dnn::Activation a96(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c97(&net, 512, 3, 3, 1, 1, 1, 1, c97_bin, true); - tk::dnn::Activation a97(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s98(&net, &s95); - - tk::dnn::Conv2d c99(&net, 512, 1, 1, 1, 1, 0, 0, c99_bin, true); - tk::dnn::Activation a99(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c100(&net, 512, 3, 3, 1, 1, 1, 1, c100_bin, true); - tk::dnn::Activation a100(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s101(&net, &s98); - - tk::dnn::Conv2d c102(&net, 512, 1, 1, 1, 1, 0, 0, c102_bin, true); - tk::dnn::Activation a102(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r103_layers[2] = {&a102, &a87}; - tk::dnn::Route r103(&net, r103_layers, 2); - - tk::dnn::Conv2d c104(&net, 1024, 1, 1, 1, 1, 0, 0, c104_bin, true); - tk::dnn::Activation a104(&net, tk::dnn::ACTIVATION_MISH); - - - //################ - tk::dnn::Conv2d c105(&net, 512, 1, 1, 1, 1, 0, 0, c105_bin, true); - tk::dnn::Activation a105(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c106(&net, 1024, 3, 3, 1, 1, 1, 1, c106_bin, true); - tk::dnn::Activation a106(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c107(&net, 512, 1, 1, 1, 1, 0, 0, c107_bin, true); - tk::dnn::Activation a107(&net, tk::dnn::ACTIVATION_LEAKY); - - //SPP - tk::dnn::Pooling p108(&net, 5, 5, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r109_layers[1] = {&a107}; - tk::dnn::Route r109(&net, r109_layers, 1); - - tk::dnn::Pooling p110(&net, 9, 9, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r111_layers[1] = {&a107}; - tk::dnn::Route r111(&net, r111_layers, 1); - - tk::dnn::Pooling p112(&net, 13, 13, 1, 1, 12, 12, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r113_layers[4] = {&p112, &p110, &p108, &a107}; - tk::dnn::Route r113(&net, r113_layers, 4); - //END SPP - - tk::dnn::Conv2d c114(&net, 512, 1, 1, 1, 1, 0, 0, c114_bin, true); - tk::dnn::Activation a114(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c115(&net, 1024, 3, 3, 1, 1, 1, 1, c115_bin, true); - tk::dnn::Activation a115(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c116(&net, 512, 1, 1, 1, 1, 0, 0, c116_bin, true); - tk::dnn::Activation a116(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c117(&net, 256, 1, 1, 1, 1, 0, 0, c117_bin, true); - tk::dnn::Activation a117(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u118(&net, 2); - tk::dnn::Layer *r119_layers[1] = {&a85}; - tk::dnn::Route r119(&net, r119_layers, 1); - tk::dnn::Conv2d c120(&net, 256, 1, 1, 1, 1, 0, 0, c120_bin, true); - tk::dnn::Activation a120(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r121_layers[2] = {&a120,&u118}; - tk::dnn::Route r121(&net, r121_layers, 2); - - tk::dnn::Conv2d c122(&net, 256, 1, 1, 1, 1, 0, 0, c122_bin, true); - tk::dnn::Activation a122(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c123(&net, 512, 3, 3, 1, 1, 1, 1, c123_bin, true); - tk::dnn::Activation a123(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c124(&net, 256, 1, 1, 1, 1, 0, 0, c124_bin, true); - tk::dnn::Activation a124(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c125(&net, 512, 3, 3, 1, 1, 1, 1, c125_bin, true); - tk::dnn::Activation a125(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c126(&net, 256, 1, 1, 1, 1, 0, 0, c126_bin, true); - tk::dnn::Activation a126(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c127(&net, 128, 1, 1, 1, 1, 0, 0, c127_bin, true); - tk::dnn::Activation a127(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u128(&net, 2); - tk::dnn::Layer *r129_layers[1] = {&a54}; - tk::dnn::Route r129(&net, r129_layers, 1); - tk::dnn::Conv2d c130(&net, 128, 1, 1, 1, 1, 0, 0, c130_bin, true); - tk::dnn::Activation a130(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r131_layers[2] = {&a130,&u128}; - tk::dnn::Route r131(&net, r131_layers, 2); - - - tk::dnn::Conv2d c132(&net, 128, 1, 1, 1, 1, 0, 0, c132_bin, true); - tk::dnn::Activation a132(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c133(&net, 256, 3, 3, 1, 1, 1, 1, c133_bin, true); - tk::dnn::Activation a133(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c134(&net, 128, 1, 1, 1, 1, 0, 0, c134_bin, true); - tk::dnn::Activation a134(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c135(&net, 256, 3, 3, 1, 1, 1, 1, c135_bin, true); - tk::dnn::Activation a135(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c136(&net, 128, 1, 1, 1, 1, 0, 0, c136_bin, true); - tk::dnn::Activation a136(&net, tk::dnn::ACTIVATION_LEAKY); - - - tk::dnn::Conv2d c137(&net, 256, 3, 3, 1, 1, 1, 1, c137_bin, true); - tk::dnn::Activation a137(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c138(&net, 255, 1, 1, 1, 1, 0, 0, c138_bin, false); - tk::dnn::Yolo yolo139(&net, classes, 3, g139_bin, 3, 1.2); - - tk::dnn::Layer *r140_layers[1] = {&a136}; - tk::dnn::Route r140(&net, r140_layers, 1); - tk::dnn::Conv2d c141(&net, 256, 3, 3, 2, 2, 1, 1, c141_bin, true); - tk::dnn::Activation a141(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r142_layers[2] = {&a141,&a126}; - tk::dnn::Route r142(&net, r142_layers, 2); - - tk::dnn::Conv2d c143(&net, 256, 1, 1, 1, 1, 0, 0, c143_bin, true); - tk::dnn::Activation a143(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c144(&net, 512, 3, 3, 1, 1, 1, 1, c144_bin, true); - tk::dnn::Activation a144(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c145(&net, 256, 1, 1, 1, 1, 0, 0, c145_bin, true); - tk::dnn::Activation a145(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c146(&net, 512, 3, 3, 1, 1, 1, 1, c146_bin, true); - tk::dnn::Activation a146(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c147(&net, 256, 1, 1, 1, 1, 0, 0, c147_bin, true); - tk::dnn::Activation a147(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c148(&net, 512, 3, 3, 1, 1, 1, 1, c148_bin, true); - tk::dnn::Activation a148(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c149(&net, 255, 1, 1, 1, 1, 0, 0, c149_bin, false); - tk::dnn::Yolo yolo150(&net, classes, 3, g150_bin, 3, 1.1); - - tk::dnn::Layer *r151_layers[1] = {&a147}; - tk::dnn::Route r151(&net, r151_layers, 1); - tk::dnn::Conv2d c152(&net, 512, 3, 3, 2, 2, 1, 1, c152_bin, true); - tk::dnn::Activation a152(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r153_layers[2] = {&a152,&a116}; - tk::dnn::Route r153(&net, r153_layers, 2); - - tk::dnn::Conv2d c154(&net, 512, 1, 1, 1, 1, 0, 0, c154_bin, true); - tk::dnn::Activation a154(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c155(&net, 1024, 3, 3, 1, 1, 1, 1, c155_bin, true); - tk::dnn::Activation a155(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c156(&net, 512, 1, 1, 1, 1, 0, 0, c156_bin, true); - tk::dnn::Activation a156(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c157(&net, 1024, 3, 3, 1, 1, 1, 1, c157_bin, true); - tk::dnn::Activation a157(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c158(&net, 512, 1, 1, 1, 1, 0, 0, c158_bin, true); - tk::dnn::Activation a158(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c159(&net, 1024, 3, 3, 1, 1, 1, 1, c159_bin, true); - tk::dnn::Activation a159(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c160(&net, 255, 1, 1, 1, 1, 0, 0, c160_bin, false); - tk::dnn::Yolo yolo161(&net, classes, 3, g161_bin, 3, 1.05); - - - - - - - yolo[0] = &yolo139; - yolo[1] = &yolo150; - yolo[2] = &yolo161; - - // fill classes names - for (int i = 0; i < 3; i++) - { - yolo[i]->classesNames = {"person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"}; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - // //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo4_512")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for (int i = 0; i < 3; i++) - out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); - { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - for (int i = 0; i < 3; i++) - cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for (int i = 0; i < 3; i++) - yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for (int j = 0; j < ndets; j++) - { - tk::dnn::Yolo::box b = dets[j].bbox; - int x0 = (b.x - b.w / 2.); - int x1 = (b.x + b.w / 2.); - int y0 = (b.y - b.h / 2.); - int y1 = (b.y + b.h / 2.); - - int cl = 0; - for (int c = 0; c < classes; ++c) - { - float prob = dets[j].prob[c]; - if (prob > 0) - cl = c; - } - std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; - } - TIMER_STOP - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); - { - dim2.print(); - TIMER_START - netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - for (int i = 0; i < 3; i++) - rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; - - int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; - for (int i = 0; i < 3; i++) - { - printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); - dnnType *out, *out_h; - int odim = out_dim[i].tot(); - readBinaryFile(output_bins[i], odim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - } - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo4/yolo4_608.cpp b/tests/yolo4/yolo4_608.cpp deleted file mode 100644 index 64d8f34..0000000 --- a/tests/yolo4/yolo4_608.cpp +++ /dev/null @@ -1,666 +0,0 @@ -#include -#include -#include "tkdnn.h" - -int main() -{ - - // Network layout - tk::dnn::dataDim_t dim(1, 3, 608, 608, 1); - tk::dnn::Network net(dim); - - // create yolo4_608 model - std::string bin_path = "yolo4_608"; - int classes = 80; - tk::dnn::Yolo *yolo[3]; - - std::string input_bin = bin_path + "/layers/input.bin"; - - std::vector output_bins = { - bin_path + "/debug/layer139_out.bin", - bin_path + "/debug/layer150_out.bin", - bin_path + "/debug/layer161_out.bin"}; - std::string c0_bin = bin_path + "/layers/c0.bin"; - std::string c1_bin = bin_path + "/layers/c1.bin"; - std::string c2_bin = bin_path + "/layers/c2.bin"; - std::string c3_bin = bin_path + "/layers/c3.bin"; - std::string c4_bin = bin_path + "/layers/c4.bin"; - std::string c5_bin = bin_path + "/layers/c5.bin"; - std::string c6_bin = bin_path + "/layers/c6.bin"; - std::string c7_bin = bin_path + "/layers/c7.bin"; - std::string c8_bin = bin_path + "/layers/c8.bin"; - std::string c10_bin = bin_path + "/layers/c10.bin"; - std::string c11_bin = bin_path + "/layers/c11.bin"; - std::string c12_bin = bin_path + "/layers/c12.bin"; - std::string c13_bin = bin_path + "/layers/c13.bin"; - std::string c14_bin = bin_path + "/layers/c14.bin"; - std::string c15_bin = bin_path + "/layers/c15.bin"; - std::string c16_bin = bin_path + "/layers/c16.bin"; - std::string c17_bin = bin_path + "/layers/c17.bin"; - std::string c18_bin = bin_path + "/layers/c18.bin"; - std::string c19_bin = bin_path + "/layers/c19.bin"; - std::string c20_bin = bin_path + "/layers/c20.bin"; - std::string c21_bin = bin_path + "/layers/c21.bin"; - std::string c23_bin = bin_path + "/layers/c23.bin"; - std::string c24_bin = bin_path + "/layers/c24.bin"; - std::string c25_bin = bin_path + "/layers/c25.bin"; - std::string c26_bin = bin_path + "/layers/c26.bin"; - std::string c27_bin = bin_path + "/layers/c27.bin"; - std::string c28_bin = bin_path + "/layers/c28.bin"; - std::string c29_bin = bin_path + "/layers/c29.bin"; - std::string c30_bin = bin_path + "/layers/c30.bin"; - std::string c31_bin = bin_path + "/layers/c31.bin"; - std::string c32_bin = bin_path + "/layers/c32.bin"; - std::string c33_bin = bin_path + "/layers/c33.bin"; - std::string c34_bin = bin_path + "/layers/c34.bin"; - std::string c35_bin = bin_path + "/layers/c35.bin"; - std::string c36_bin = bin_path + "/layers/c36.bin"; - std::string c37_bin = bin_path + "/layers/c37.bin"; - std::string c38_bin = bin_path + "/layers/c38.bin"; - std::string c39_bin = bin_path + "/layers/c39.bin"; - std::string c40_bin = bin_path + "/layers/c40.bin"; - std::string c41_bin = bin_path + "/layers/c41.bin"; - std::string c42_bin = bin_path + "/layers/c42.bin"; - std::string c43_bin = bin_path + "/layers/c43.bin"; - std::string c44_bin = bin_path + "/layers/c44.bin"; - std::string c45_bin = bin_path + "/layers/c45.bin"; - std::string c46_bin = bin_path + "/layers/c46.bin"; - std::string c47_bin = bin_path + "/layers/c47.bin"; - std::string c48_bin = bin_path + "/layers/c48.bin"; - std::string c49_bin = bin_path + "/layers/c49.bin"; - std::string c50_bin = bin_path + "/layers/c50.bin"; - std::string c51_bin = bin_path + "/layers/c51.bin"; - std::string c52_bin = bin_path + "/layers/c52.bin"; - std::string c53_bin = bin_path + "/layers/c53.bin"; - std::string c54_bin = bin_path + "/layers/c54.bin"; - std::string c55_bin = bin_path + "/layers/c55.bin"; - std::string c56_bin = bin_path + "/layers/c56.bin"; - std::string c57_bin = bin_path + "/layers/c57.bin"; - std::string c58_bin = bin_path + "/layers/c58.bin"; - std::string c59_bin = bin_path + "/layers/c59.bin"; - std::string c60_bin = bin_path + "/layers/c60.bin"; - std::string c61_bin = bin_path + "/layers/c61.bin"; - std::string c62_bin = bin_path + "/layers/c62.bin"; - std::string c63_bin = bin_path + "/layers/c63.bin"; - std::string c65_bin = bin_path + "/layers/c65.bin"; - std::string c66_bin = bin_path + "/layers/c66.bin"; - std::string c67_bin = bin_path + "/layers/c67.bin"; - std::string c68_bin = bin_path + "/layers/c68.bin"; - std::string c69_bin = bin_path + "/layers/c69.bin"; - std::string c70_bin = bin_path + "/layers/c70.bin"; - std::string c71_bin = bin_path + "/layers/c71.bin"; - std::string c72_bin = bin_path + "/layers/c72.bin"; - std::string c74_bin = bin_path + "/layers/c74.bin"; - std::string c75_bin = bin_path + "/layers/c75.bin"; - std::string c76_bin = bin_path + "/layers/c76.bin"; - std::string c77_bin = bin_path + "/layers/c77.bin"; - std::string c78_bin = bin_path + "/layers/c78.bin"; - std::string c80_bin = bin_path + "/layers/c80.bin"; - std::string c81_bin = bin_path + "/layers/c81.bin"; - std::string c82_bin = bin_path + "/layers/c82.bin"; - std::string c83_bin = bin_path + "/layers/c83.bin"; - std::string c85_bin = bin_path + "/layers/c85.bin"; - std::string c86_bin = bin_path + "/layers/c86.bin"; - std::string c87_bin = bin_path + "/layers/c87.bin"; - std::string c89_bin = bin_path + "/layers/c89.bin"; - std::string c90_bin = bin_path + "/layers/c90.bin"; - std::string c91_bin = bin_path + "/layers/c91.bin"; - std::string c92_bin = bin_path + "/layers/c92.bin"; - std::string c93_bin = bin_path + "/layers/c93.bin"; - std::string c94_bin = bin_path + "/layers/c94.bin"; - std::string c96_bin = bin_path + "/layers/c96.bin"; - std::string c97_bin = bin_path + "/layers/c97.bin"; - std::string c98_bin = bin_path + "/layers/c98.bin"; - std::string c99_bin = bin_path + "/layers/c99.bin"; - std::string c100_bin = bin_path + "/layers/c100.bin"; - std::string c101_bin = bin_path + "/layers/c101.bin"; - std::string c102_bin = bin_path + "/layers/c102.bin"; - std::string c103_bin = bin_path + "/layers/c103.bin"; - std::string c104_bin = bin_path + "/layers/c104.bin"; - std::string c105_bin = bin_path + "/layers/c105.bin"; - std::string c106_bin = bin_path + "/layers/c106.bin"; - std::string c107_bin = bin_path + "/layers/c107.bin"; - std::string c108_bin = bin_path + "/layers/c108.bin"; - std::string c109_bin = bin_path + "/layers/c109.bin"; - std::string c110_bin = bin_path + "/layers/c110.bin"; - std::string c111_bin = bin_path + "/layers/c111.bin"; - std::string c112_bin = bin_path + "/layers/c112.bin"; - std::string c113_bin = bin_path + "/layers/c113.bin"; - std::string c114_bin = bin_path + "/layers/c114.bin"; - std::string c115_bin = bin_path + "/layers/c115.bin"; - std::string c116_bin = bin_path + "/layers/c116.bin"; - std::string c117_bin = bin_path + "/layers/c117.bin"; - std::string c119_bin = bin_path + "/layers/c119.bin"; - std::string c120_bin = bin_path + "/layers/c120.bin"; - std::string c121_bin = bin_path + "/layers/c121.bin"; - std::string c122_bin = bin_path + "/layers/c122.bin"; - std::string c123_bin = bin_path + "/layers/c123.bin"; - std::string c124_bin = bin_path + "/layers/c124.bin"; - std::string c125_bin = bin_path + "/layers/c125.bin"; - std::string c126_bin = bin_path + "/layers/c126.bin"; - std::string c127_bin = bin_path + "/layers/c127.bin"; - std::string c128_bin = bin_path + "/layers/c128.bin"; - std::string c130_bin = bin_path + "/layers/c130.bin"; - std::string c131_bin = bin_path + "/layers/c131.bin"; - std::string c132_bin = bin_path + "/layers/c132.bin"; - std::string c133_bin = bin_path + "/layers/c133.bin"; - std::string c134_bin = bin_path + "/layers/c134.bin"; - std::string c135_bin = bin_path + "/layers/c135.bin"; - std::string c136_bin = bin_path + "/layers/c136.bin"; - std::string c137_bin = bin_path + "/layers/c137.bin"; - std::string c138_bin = bin_path + "/layers/c138.bin"; - std::string c141_bin = bin_path + "/layers/c141.bin"; - std::string c142_bin = bin_path + "/layers/c142.bin"; - std::string c143_bin = bin_path + "/layers/c143.bin"; - std::string c144_bin = bin_path + "/layers/c144.bin"; - std::string c145_bin = bin_path + "/layers/c145.bin"; - std::string c146_bin = bin_path + "/layers/c146.bin"; - std::string c147_bin = bin_path + "/layers/c147.bin"; - std::string c148_bin = bin_path + "/layers/c148.bin"; - std::string c149_bin = bin_path + "/layers/c149.bin"; - std::string c150_bin = bin_path + "/layers/c150.bin"; - std::string c151_bin = bin_path + "/layers/c151.bin"; - std::string c152_bin = bin_path + "/layers/c152.bin"; - std::string c153_bin = bin_path + "/layers/c153.bin"; - std::string c154_bin = bin_path + "/layers/c154.bin"; - std::string c155_bin = bin_path + "/layers/c155.bin"; - std::string c156_bin = bin_path + "/layers/c156.bin"; - std::string c157_bin = bin_path + "/layers/c157.bin"; - std::string c158_bin = bin_path + "/layers/c158.bin"; - std::string c159_bin = bin_path + "/layers/c159.bin"; - std::string c160_bin = bin_path + "/layers/c160.bin"; - std::string g139_bin = bin_path + "/layers/g139.bin"; - std::string g150_bin = bin_path + "/layers/g150.bin"; - std::string g161_bin = bin_path + "/layers/g161.bin"; - - - downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); - - tk::dnn::Conv2d c0(&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); - tk::dnn::Activation a0(&net, tk::dnn::ACTIVATION_MISH); - - // downsample - tk::dnn::Conv2d c1(&net, 64, 3, 3, 2, 2, 1, 1, c1_bin, true); - tk::dnn::Activation a1(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c2(&net, 64, 1, 1, 1, 1, 0, 0, c2_bin, true); - tk::dnn::Activation a2(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r3_layers[1] = {&a1}; - tk::dnn::Route r3(&net, r3_layers, 1); - - tk::dnn::Conv2d c4(&net, 64, 1, 1, 1, 1, 0, 0, c4_bin, true); - tk::dnn::Activation a4(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c5(&net, 32, 1, 1, 1, 1, 0, 0, c5_bin, true); - tk::dnn::Activation a5(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c6(&net, 64, 3, 3, 1, 1, 1, 1, c6_bin, true); - tk::dnn::Activation a6(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s7(&net, &a4); - - tk::dnn::Conv2d c8(&net, 64, 1, 1, 1, 1, 0, 0, c8_bin, true); - tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r9_layers[2] = {&a8, &a2}; - tk::dnn::Route r9(&net, r9_layers, 2); - - tk::dnn::Conv2d c10(&net, 64, 1, 1, 1, 1, 0, 0, c10_bin, true); - tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_MISH); - - // downsample - tk::dnn::Conv2d c11(&net, 128, 3, 3, 2, 2, 1, 1, c11_bin, true); - tk::dnn::Activation a11(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c12(&net, 64, 1, 1, 1, 1, 0, 0, c12_bin, true); - tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r13_layers[1] = {&a11}; - tk::dnn::Route r13(&net, r13_layers, 1); - - tk::dnn::Conv2d c14(&net, 64, 1, 1, 1, 1, 0, 0, c14_bin, true); - tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c15(&net, 64, 1, 1, 1, 1, 0, 0, c15_bin, true); - tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c16(&net, 64, 3, 3, 1, 1, 1, 1, c16_bin, true); - tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s17(&net, &a14); - - tk::dnn::Conv2d c18(&net, 64, 1, 1, 1, 1, 0, 0, c18_bin, true); - tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c19(&net, 64, 3, 3, 1, 1, 1, 1, c19_bin, true); - tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s20(&net, &s17); - - tk::dnn::Conv2d c21(&net, 64, 1, 1, 1, 1, 0, 0, c21_bin, true); - tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r22_layers[2] = {&a21, &a12}; - tk::dnn::Route r22(&net, r22_layers, 2); - - tk::dnn::Conv2d c23(&net, 128, 1, 1, 1, 1, 0, 0, c23_bin, true); - tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c24(&net, 256, 3, 3, 2, 2, 1, 1, c24_bin, true); - tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c25(&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true); - tk::dnn::Activation a25(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r26_layers[1] = {&a24}; - tk::dnn::Route r26(&net, r26_layers, 1); - - tk::dnn::Conv2d c27(&net, 128, 1, 1, 1, 1, 0, 0, c27_bin, true); - tk::dnn::Activation a27(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c28(&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true); - tk::dnn::Activation a28(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c29(&net, 128, 3, 3, 1, 1, 1, 1, c29_bin, true); - tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s30(&net, &a27); - - tk::dnn::Conv2d c31(&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true); - tk::dnn::Activation a31(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c32(&net, 128, 3, 3, 1, 1, 1, 1, c32_bin, true); - tk::dnn::Activation a32(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s33(&net, &s30); - - tk::dnn::Conv2d c34(&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true); - tk::dnn::Activation a34(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c35(&net, 128, 3, 3, 1, 1, 1, 1, c35_bin, true); - tk::dnn::Activation a35(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s36(&net, &s33); - - tk::dnn::Conv2d c37(&net, 128, 1, 1, 1, 1, 0, 0, c37_bin, true); - tk::dnn::Activation a37(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c38(&net, 128, 3, 3, 1, 1, 1, 1, c38_bin, true); - tk::dnn::Activation a38(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s39(&net, &s36); - - tk::dnn::Conv2d c40(&net, 128, 1, 1, 1, 1, 0, 0, c40_bin, true); - tk::dnn::Activation a40(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c41(&net, 128, 3, 3, 1, 1, 1, 1, c41_bin, true); - tk::dnn::Activation a41(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s42(&net, &s39); - - tk::dnn::Conv2d c43(&net, 128, 1, 1, 1, 1, 0, 0, c43_bin, true); - tk::dnn::Activation a43(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c44(&net, 128, 3, 3, 1, 1, 1, 1, c44_bin, true); - tk::dnn::Activation a44(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s45(&net, &s42); - - tk::dnn::Conv2d c46(&net, 128, 1, 1, 1, 1, 0, 0, c46_bin, true); - tk::dnn::Activation a46(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c47(&net, 128, 3, 3, 1, 1, 1, 1, c47_bin, true); - tk::dnn::Activation a47(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s48(&net, &s45); - - tk::dnn::Conv2d c49(&net, 128, 1, 1, 1, 1, 0, 0, c49_bin, true); - tk::dnn::Activation a49(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c50(&net, 128, 3, 3, 1, 1, 1, 1, c50_bin, true); - tk::dnn::Activation a50(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s51(&net, &s48); - - tk::dnn::Conv2d c52(&net, 128, 1, 1, 1, 1, 0, 0, c52_bin, true); - tk::dnn::Activation a52(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r53_layers[2] = {&a52, &a25}; - tk::dnn::Route r53(&net, r53_layers, 2); - - tk::dnn::Conv2d c54(&net, 256, 1, 1, 1, 1, 0, 0, c54_bin, true); - tk::dnn::Activation a54(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c55(&net, 512, 3, 3, 2, 2, 1, 1, c55_bin, true); - tk::dnn::Activation a55(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c56(&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true); - tk::dnn::Activation a56(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r57_layers[1] = {&a55}; - tk::dnn::Route r57(&net, r57_layers, 1); - - tk::dnn::Conv2d c58(&net, 256, 1, 1, 1, 1, 0, 0, c58_bin, true); - tk::dnn::Activation a58(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c59(&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true); - tk::dnn::Activation a59(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c60(&net, 256, 3, 3, 1, 1, 1, 1, c60_bin, true); - tk::dnn::Activation a60(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s61(&net, &a58); - - tk::dnn::Conv2d c62(&net, 256, 1, 1, 1, 1, 0, 0, c62_bin, true); - tk::dnn::Activation a62(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c63(&net, 256, 3, 3, 1, 1, 1, 1, c63_bin, true); - tk::dnn::Activation a63(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s64(&net, &s61); - - tk::dnn::Conv2d c65(&net, 256, 1, 1, 1, 1, 0, 0, c65_bin, true); - tk::dnn::Activation a65(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c66(&net, 256, 3, 3, 1, 1, 1, 1, c66_bin, true); - tk::dnn::Activation a66(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s67(&net, &s64); - - tk::dnn::Conv2d c68(&net, 256, 1, 1, 1, 1, 0, 0, c68_bin, true); - tk::dnn::Activation a68(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c69(&net, 256, 3, 3, 1, 1, 1, 1, c69_bin, true); - tk::dnn::Activation a69(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s70(&net, &s67); - - tk::dnn::Conv2d c71(&net, 256, 1, 1, 1, 1, 0, 0, c71_bin, true); - tk::dnn::Activation a71(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c72(&net, 256, 3, 3, 1, 1, 1, 1, c72_bin, true); - tk::dnn::Activation a72(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s73(&net, &s70); - - tk::dnn::Conv2d c74(&net, 256, 1, 1, 1, 1, 0, 0, c74_bin, true); - tk::dnn::Activation a74(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c75(&net, 256, 3, 3, 1, 1, 1, 1, c75_bin, true); - tk::dnn::Activation a75(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s76(&net, &s73); - - tk::dnn::Conv2d c77(&net, 256, 1, 1, 1, 1, 0, 0, c77_bin, true); - tk::dnn::Activation a77(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c78(&net, 256, 3, 3, 1, 1, 1, 1, c78_bin, true); - tk::dnn::Activation a78(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s79(&net, &s76); - - tk::dnn::Conv2d c80(&net, 256, 1, 1, 1, 1, 0, 0, c80_bin, true); - tk::dnn::Activation a80(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c81(&net, 256, 3, 3, 1, 1, 1, 1, c81_bin, true); - tk::dnn::Activation a81(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s82(&net, &s79); - - tk::dnn::Conv2d c83(&net, 256, 1, 1, 1, 1, 0, 0, c83_bin, true); - tk::dnn::Activation a83(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r84_layers[2] = {&a83, &a56}; - tk::dnn::Route r84(&net, r84_layers, 2); - - tk::dnn::Conv2d c85(&net, 512, 1, 1, 1, 1, 0, 0, c85_bin, true); - tk::dnn::Activation a85(&net, tk::dnn::ACTIVATION_MISH); - - //downsample - tk::dnn::Conv2d c86(&net, 1024, 3, 3, 2, 2, 1, 1, c86_bin, true); - tk::dnn::Activation a86(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c87(&net, 512, 1, 1, 1, 1, 0, 0, c87_bin, true); - tk::dnn::Activation a87(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r88_layers[1] = {&a86}; - tk::dnn::Route r88(&net, r88_layers, 1); - - tk::dnn::Conv2d c89(&net, 512, 1, 1, 1, 1, 0, 0, c89_bin, true); - tk::dnn::Activation a89(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c90(&net, 512, 1, 1, 1, 1, 0, 0, c90_bin, true); - tk::dnn::Activation a90(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c91(&net, 512, 3, 3, 1, 1, 1, 1, c91_bin, true); - tk::dnn::Activation a91(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s92(&net, &a89); - - tk::dnn::Conv2d c93(&net, 512, 1, 1, 1, 1, 0, 0, c93_bin, true); - tk::dnn::Activation a93(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c94(&net, 512, 3, 3, 1, 1, 1, 1, c94_bin, true); - tk::dnn::Activation a94(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s95(&net, &s92); - - tk::dnn::Conv2d c96(&net, 512, 1, 1, 1, 1, 0, 0, c96_bin, true); - tk::dnn::Activation a96(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c97(&net, 512, 3, 3, 1, 1, 1, 1, c97_bin, true); - tk::dnn::Activation a97(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s98(&net, &s95); - - tk::dnn::Conv2d c99(&net, 512, 1, 1, 1, 1, 0, 0, c99_bin, true); - tk::dnn::Activation a99(&net, tk::dnn::ACTIVATION_MISH); - tk::dnn::Conv2d c100(&net, 512, 3, 3, 1, 1, 1, 1, c100_bin, true); - tk::dnn::Activation a100(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Shortcut s101(&net, &s98); - - tk::dnn::Conv2d c102(&net, 512, 1, 1, 1, 1, 0, 0, c102_bin, true); - tk::dnn::Activation a102(&net, tk::dnn::ACTIVATION_MISH); - - tk::dnn::Layer *r103_layers[2] = {&a102, &a87}; - tk::dnn::Route r103(&net, r103_layers, 2); - - tk::dnn::Conv2d c104(&net, 1024, 1, 1, 1, 1, 0, 0, c104_bin, true); - tk::dnn::Activation a104(&net, tk::dnn::ACTIVATION_MISH); - - - //################ - tk::dnn::Conv2d c105(&net, 512, 1, 1, 1, 1, 0, 0, c105_bin, true); - tk::dnn::Activation a105(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c106(&net, 1024, 3, 3, 1, 1, 1, 1, c106_bin, true); - tk::dnn::Activation a106(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c107(&net, 512, 1, 1, 1, 1, 0, 0, c107_bin, true); - tk::dnn::Activation a107(&net, tk::dnn::ACTIVATION_LEAKY); - - //SPP - tk::dnn::Pooling p108(&net, 5, 5, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r109_layers[1] = {&a107}; - tk::dnn::Route r109(&net, r109_layers, 1); - - tk::dnn::Pooling p110(&net, 9, 9, 1, 1, 0, 0, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r111_layers[1] = {&a107}; - tk::dnn::Route r111(&net, r111_layers, 1); - - tk::dnn::Pooling p112(&net, 13, 13, 1, 1, 12, 12, tk::dnn::POOLING_MAX_FIXEDSIZE); - tk::dnn::Layer *r113_layers[4] = {&p112, &p110, &p108, &a107}; - tk::dnn::Route r113(&net, r113_layers, 4); - //END SPP - - tk::dnn::Conv2d c114(&net, 512, 1, 1, 1, 1, 0, 0, c114_bin, true); - tk::dnn::Activation a114(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c115(&net, 1024, 3, 3, 1, 1, 1, 1, c115_bin, true); - tk::dnn::Activation a115(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c116(&net, 512, 1, 1, 1, 1, 0, 0, c116_bin, true); - tk::dnn::Activation a116(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c117(&net, 256, 1, 1, 1, 1, 0, 0, c117_bin, true); - tk::dnn::Activation a117(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u118(&net, 2); - tk::dnn::Layer *r119_layers[1] = {&a85}; - tk::dnn::Route r119(&net, r119_layers, 1); - tk::dnn::Conv2d c120(&net, 256, 1, 1, 1, 1, 0, 0, c120_bin, true); - tk::dnn::Activation a120(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r121_layers[2] = {&a120,&u118}; - tk::dnn::Route r121(&net, r121_layers, 2); - - tk::dnn::Conv2d c122(&net, 256, 1, 1, 1, 1, 0, 0, c122_bin, true); - tk::dnn::Activation a122(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c123(&net, 512, 3, 3, 1, 1, 1, 1, c123_bin, true); - tk::dnn::Activation a123(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c124(&net, 256, 1, 1, 1, 1, 0, 0, c124_bin, true); - tk::dnn::Activation a124(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c125(&net, 512, 3, 3, 1, 1, 1, 1, c125_bin, true); - tk::dnn::Activation a125(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c126(&net, 256, 1, 1, 1, 1, 0, 0, c126_bin, true); - tk::dnn::Activation a126(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c127(&net, 128, 1, 1, 1, 1, 0, 0, c127_bin, true); - tk::dnn::Activation a127(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Upsample u128(&net, 2); - tk::dnn::Layer *r129_layers[1] = {&a54}; - tk::dnn::Route r129(&net, r129_layers, 1); - tk::dnn::Conv2d c130(&net, 128, 1, 1, 1, 1, 0, 0, c130_bin, true); - tk::dnn::Activation a130(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r131_layers[2] = {&a130,&u128}; - tk::dnn::Route r131(&net, r131_layers, 2); - - - tk::dnn::Conv2d c132(&net, 128, 1, 1, 1, 1, 0, 0, c132_bin, true); - tk::dnn::Activation a132(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c133(&net, 256, 3, 3, 1, 1, 1, 1, c133_bin, true); - tk::dnn::Activation a133(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c134(&net, 128, 1, 1, 1, 1, 0, 0, c134_bin, true); - tk::dnn::Activation a134(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c135(&net, 256, 3, 3, 1, 1, 1, 1, c135_bin, true); - tk::dnn::Activation a135(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c136(&net, 128, 1, 1, 1, 1, 0, 0, c136_bin, true); - tk::dnn::Activation a136(&net, tk::dnn::ACTIVATION_LEAKY); - - - tk::dnn::Conv2d c137(&net, 256, 3, 3, 1, 1, 1, 1, c137_bin, true); - tk::dnn::Activation a137(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c138(&net, 255, 1, 1, 1, 1, 0, 0, c138_bin, false); - tk::dnn::Yolo yolo139(&net, classes, 3, g139_bin, 3, 1.2); - - tk::dnn::Layer *r140_layers[1] = {&a136}; - tk::dnn::Route r140(&net, r140_layers, 1); - tk::dnn::Conv2d c141(&net, 256, 3, 3, 2, 2, 1, 1, c141_bin, true); - tk::dnn::Activation a141(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r142_layers[2] = {&a141,&a126}; - tk::dnn::Route r142(&net, r142_layers, 2); - - tk::dnn::Conv2d c143(&net, 256, 1, 1, 1, 1, 0, 0, c143_bin, true); - tk::dnn::Activation a143(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c144(&net, 512, 3, 3, 1, 1, 1, 1, c144_bin, true); - tk::dnn::Activation a144(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c145(&net, 256, 1, 1, 1, 1, 0, 0, c145_bin, true); - tk::dnn::Activation a145(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c146(&net, 512, 3, 3, 1, 1, 1, 1, c146_bin, true); - tk::dnn::Activation a146(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c147(&net, 256, 1, 1, 1, 1, 0, 0, c147_bin, true); - tk::dnn::Activation a147(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c148(&net, 512, 3, 3, 1, 1, 1, 1, c148_bin, true); - tk::dnn::Activation a148(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c149(&net, 255, 1, 1, 1, 1, 0, 0, c149_bin, false); - tk::dnn::Yolo yolo150(&net, classes, 3, g150_bin, 3, 1.1); - - tk::dnn::Layer *r151_layers[1] = {&a147}; - tk::dnn::Route r151(&net, r151_layers, 1); - tk::dnn::Conv2d c152(&net, 512, 3, 3, 2, 2, 1, 1, c152_bin, true); - tk::dnn::Activation a152(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Layer *r153_layers[2] = {&a152,&a116}; - tk::dnn::Route r153(&net, r153_layers, 2); - - tk::dnn::Conv2d c154(&net, 512, 1, 1, 1, 1, 0, 0, c154_bin, true); - tk::dnn::Activation a154(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c155(&net, 1024, 3, 3, 1, 1, 1, 1, c155_bin, true); - tk::dnn::Activation a155(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c156(&net, 512, 1, 1, 1, 1, 0, 0, c156_bin, true); - tk::dnn::Activation a156(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c157(&net, 1024, 3, 3, 1, 1, 1, 1, c157_bin, true); - tk::dnn::Activation a157(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c158(&net, 512, 1, 1, 1, 1, 0, 0, c158_bin, true); - tk::dnn::Activation a158(&net, tk::dnn::ACTIVATION_LEAKY); - - tk::dnn::Conv2d c159(&net, 1024, 3, 3, 1, 1, 1, 1, c159_bin, true); - tk::dnn::Activation a159(&net, tk::dnn::ACTIVATION_LEAKY); - tk::dnn::Conv2d c160(&net, 255, 1, 1, 1, 1, 0, 0, c160_bin, false); - tk::dnn::Yolo yolo161(&net, classes, 3, g161_bin, 3, 1.05); - - - - - - - yolo[0] = &yolo139; - yolo[1] = &yolo150; - yolo[2] = &yolo161; - - // fill classes names - for (int i = 0; i < 3; i++) - { - yolo[i]->classesNames = {"person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"}; - } - - // Load input - dnnType *data; - dnnType *input_h; - readBinaryFile(input_bin, dim.tot(), &input_h, &data); - - //print network model - net.print(); - - // //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo4_608")); - - // the network have 3 outputs - tk::dnn::dataDim_t out_dim[3]; - for (int i = 0; i < 3; i++) - out_dim[i] = yolo[i]->output_dim; - dnnType *cudnn_out[3], *rt_out[3]; - - tk::dnn::dataDim_t dim1 = dim; //input dim - printCenteredTitle(" CUDNN inference ", '=', 30); - { - dim1.print(); - TIMER_START - net.infer(dim1, data); - TIMER_STOP - dim1.print(); - } - - for (int i = 0; i < 3; i++) - cudnn_out[i] = yolo[i]->dstData; - - printCenteredTitle(" compute detections ", '=', 30); - TIMER_START - int ndets = 0; - tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - for (int i = 0; i < 3; i++) - yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5); - tk::dnn::Yolo::mergeDetections(dets, ndets, classes); - - for (int j = 0; j < ndets; j++) - { - tk::dnn::Yolo::box b = dets[j].bbox; - int x0 = (b.x - b.w / 2.); - int x1 = (b.x + b.w / 2.); - int y0 = (b.y - b.h / 2.); - int y1 = (b.y + b.h / 2.); - - int cl = 0; - for (int c = 0; c < classes; ++c) - { - float prob = dets[j].prob[c]; - if (prob > 0) - cl = c; - } - std::cout << cl << ": " << x0 << " " << y0 << " " << x1 << " " << y1 << "\n"; - } - TIMER_STOP - - tk::dnn::dataDim_t dim2 = dim; - printCenteredTitle(" TENSORRT inference ", '=', 30); - { - dim2.print(); - TIMER_START - netRT.infer(dim2, data); - TIMER_STOP - dim2.print(); - } - - for (int i = 0; i < 3; i++) - rt_out[i] = (dnnType *)netRT.buffersRT[i + 1]; - - int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; - for (int i = 0; i < 3; i++) - { - printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30); - dnnType *out, *out_h; - int odim = out_dim[i].tot(); - readBinaryFile(output_bins[i], odim, &out_h, &out); - std::cout<<"CUDNN vs correct"; - ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - } - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; -} diff --git a/tests/yolo4_berkeley/yolo4_berkeley.cpp b/tests/yolo4_berkeley/yolo4_berkeley.cpp index d10ad59..f80a0ac 100644 --- a/tests/yolo4_berkeley/yolo4_berkeley.cpp +++ b/tests/yolo4_berkeley/yolo4_berkeley.cpp @@ -174,7 +174,7 @@ int main() std::string g161_bin = bin_path + "/layers/g161.bin"; - downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); + downloadWeightsifDoNotExist(input_bin, bin_path, "https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download"); tk::dnn::Conv2d c0(&net, 32, 3, 3, 1, 1, 1, 1, c0_bin, true); tk::dnn::Activation a0(&net, tk::dnn::ACTIVATION_MISH); -- 2.52.0 From c8ed6d782a143e25972fc6797e044188e2d308c9 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Mon, 1 Jun 2020 16:15:29 +0200 Subject: [PATCH 036/228] all test ok --- include/tkDNN/DarknetParser.h | 12 ++++++------ include/tkDNN/DetectionNN.h | 12 ++++++------ include/tkDNN/test.h | 8 ++++---- include/tkDNN/utils.h | 6 +++--- tests/backbones/dla34/dla34.cpp | 8 ++++---- tests/backbones/resnet101/resnet101.cpp | 8 ++++---- tests/centernet/dla34_cnet/dla34_cnet.cpp | 8 ++++---- tests/centernet/resnet101_cnet/resnet101_cnet.cpp | 8 ++++---- tests/imuodom/imuodom.cpp | 4 ++-- tests/mnist/test_mnist.cpp | 8 ++++---- tests/mnist/test_mnistRT.cpp | 8 ++++---- .../bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp | 8 ++++---- tests/mobilenet/mobilenetv2ssd/mobilenetv2ssd.cpp | 8 ++++---- .../mobilenetv2ssd512/mobilenetv2ssd512.cpp | 8 ++++---- tests/simple/test_simple.cpp | 8 ++++---- tests/test_rtinference/rtinference.cpp | 4 ++-- 16 files changed, 63 insertions(+), 63 deletions(-) diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index a918a5b..cbeed48 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -124,7 +124,7 @@ namespace tk { namespace dnn { } tk::dnn::Network *darknetAddNet(darknetFields_t &fields) { - std::cout<<"Add Net: "<= netLayers.size()) FatalError("impossible to shortcut\n"); - std::cout<<"shortcut to "<getLayerName()<<"\n"; + //std::cout<<"shortcut to "<getLayerName()<<"\n"; netLayers.push_back(new tk::dnn::Shortcut(net, netLayers[layerIdx])); } else if(f.type == "upsample") { @@ -177,7 +177,7 @@ namespace tk { namespace dnn { if(layerIdx < 0) layerIdx = netLayers.size() + layerIdx; if(layerIdx < 0 || layerIdx >= netLayers.size()) FatalError("impossible to route\n"); - std::cout<<"Route to "<getLayerName()<<"\n"; + //std::cout<<"Route to "<getLayerName()<<"\n"; layers.push_back(netLayers[layerIdx]); } netLayers.push_back(new tk::dnn::Route(net, layers.data(), layers.size())); @@ -190,7 +190,7 @@ namespace tk { namespace dnn { } else if(f.type == "yolo") { std::string wgs = wgs_path + "/g" + std::to_string(netLayers.size()) + ".bin"; - printf("%d %d %s %d %f\n", f.classes, f.num/f.n_mask, wgs.c_str(), f.n_mask, f.scale_xy); + //printf("%d %d %s %d %f\n", f.classes, f.num/f.n_mask, wgs.c_str(), f.n_mask, f.scale_xy); tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy); if(names.size() != f.classes) FatalError("Mismatch between number of classes and names"); diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index 144ded5..030cf8f 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -106,14 +106,14 @@ class DetectionNN { originalSize.clear(); if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); { - TIMER_START + TKDNN_TSTART for(int bi=0; biinfer(dim, input_d); - TIMER_STOP + TKDNN_TSTOP if(TKDNN_VERBOSE) dim.print(); stats.push_back(t_ns); if(save_times) *times< input_bins, std::vector tk::dnn::dataDim_t dim1 = net->input_dim; //input dim printCenteredTitle(" CUDNN inference ", '=', 30); { dim1.print(); - TIMER_START + TKDNN_TSTART net->infer(dim1, data); - TIMER_STOP + TKDNN_TSTOP dim1.print(); } for(int i=0; idstData; @@ -45,9 +45,9 @@ int testInference(std::vector input_bins, std::vector tk::dnn::dataDim_t dim2 = net->input_dim; printCenteredTitle(" TENSORRT inference ", '=', 30); { dim2.print(); - TIMER_START + TKDNN_TSTART netRT->infer(dim2, data); - TIMER_STOP + TKDNN_TSTOP dim2.print(); } for(int i=0; ibuffersRT[i+1]; diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index cc9a4cd..f9f6ae7 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -39,15 +39,15 @@ #define TKDNN_VERBOSE 0 // Simple Timer -#define TIMER_START timespec start, end; \ +#define TKDNN_TSTART timespec start, end; \ clock_gettime(CLOCK_MONOTONIC, &start); -#define TIMER_STOP_C(col, show) clock_gettime(CLOCK_MONOTONIC, &end); \ +#define TKDNN_TSTOP_C(col, show) clock_gettime(CLOCK_MONOTONIC, &end); \ double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 + \ (double)(end.tv_nsec - start.tv_nsec))/1.0e6; \ if(show) std::cout<enqueue(1, buffers, stream, nullptr); - TIMER_STOP + TKDNN_TSTOP checkCuda(cudaMemcpyAsync(output, buffers[outputIndex],10*sizeof(float), cudaMemcpyDeviceToHost, stream)); cudaStreamSynchronize(stream); } diff --git a/tests/mobilenet/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp b/tests/mobilenet/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp index 1549983..c3c6472 100644 --- a/tests/mobilenet/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp +++ b/tests/mobilenet/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp @@ -477,9 +477,9 @@ int main() printCenteredTitle(" CUDNN inference ", '=', 30); { dim1.print(); - TIMER_START + TKDNN_TSTART net.infer(dim1, data); - TIMER_STOP + TKDNN_TSTOP dim1.print(); } @@ -492,9 +492,9 @@ int main() printCenteredTitle(" TENSORRT inference ", '=', 30); { dim2.print(); - TIMER_START + TKDNN_TSTART netRT.infer(dim2, data); - TIMER_STOP + TKDNN_TSTOP dim2.print(); } diff --git a/tests/mobilenet/mobilenetv2ssd/mobilenetv2ssd.cpp b/tests/mobilenet/mobilenetv2ssd/mobilenetv2ssd.cpp index 787341f..58463a4 100644 --- a/tests/mobilenet/mobilenetv2ssd/mobilenetv2ssd.cpp +++ b/tests/mobilenet/mobilenetv2ssd/mobilenetv2ssd.cpp @@ -477,9 +477,9 @@ int main() printCenteredTitle(" CUDNN inference ", '=', 30); { dim1.print(); - TIMER_START + TKDNN_TSTART net.infer(dim1, data); - TIMER_STOP + TKDNN_TSTOP dim1.print(); } @@ -492,9 +492,9 @@ int main() printCenteredTitle(" TENSORRT inference ", '=', 30); { dim2.print(); - TIMER_START + TKDNN_TSTART netRT.infer(dim2, data); - TIMER_STOP + TKDNN_TSTOP dim2.print(); } diff --git a/tests/mobilenet/mobilenetv2ssd512/mobilenetv2ssd512.cpp b/tests/mobilenet/mobilenetv2ssd512/mobilenetv2ssd512.cpp index 8886232..54b00c1 100644 --- a/tests/mobilenet/mobilenetv2ssd512/mobilenetv2ssd512.cpp +++ b/tests/mobilenet/mobilenetv2ssd512/mobilenetv2ssd512.cpp @@ -476,9 +476,9 @@ int main() printCenteredTitle(" CUDNN inference ", '=', 30); { dim1.print(); - TIMER_START + TKDNN_TSTART net.infer(dim1, data); - TIMER_STOP + TKDNN_TSTOP dim1.print(); } @@ -491,9 +491,9 @@ int main() printCenteredTitle(" TENSORRT inference ", '=', 30); { dim2.print(); - TIMER_START + TKDNN_TSTART netRT.infer(dim2, data); - TIMER_STOP + TKDNN_TSTOP dim2.print(); } diff --git a/tests/simple/test_simple.cpp b/tests/simple/test_simple.cpp index b2b0441..10b0d2f 100644 --- a/tests/simple/test_simple.cpp +++ b/tests/simple/test_simple.cpp @@ -38,18 +38,18 @@ int main() { tk::dnn::dataDim_t dim1 = dim; //input dim printCenteredTitle(" CUDNN inference ", '=', 30); { dim1.print(); - TIMER_START + TKDNN_TSTART out_data = net.infer(dim1, data); - TIMER_STOP + TKDNN_TSTOP dim1.print(); } tk::dnn::dataDim_t dim2 = dim; printCenteredTitle(" TENSORRT inference ", '=', 30); { dim2.print(); - TIMER_START + TKDNN_TSTART out_data2 = netRT.infer(dim2, data); - TIMER_STOP + TKDNN_TSTOP dim2.print(); } diff --git a/tests/test_rtinference/rtinference.cpp b/tests/test_rtinference/rtinference.cpp index 4b6b21f..a629168 100644 --- a/tests/test_rtinference/rtinference.cpp +++ b/tests/test_rtinference/rtinference.cpp @@ -42,9 +42,9 @@ int main(int argc, char *argv[]) { checkCuda(cudaMemcpy(input_d, input, idim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice)); tk::dnn::dataDim_t dim = idim; - TIMER_START + TKDNN_TSTART netRT.infer(dim, input_d); - TIMER_STOP + TKDNN_TSTOP total_time+= t_ns; // control output -- 2.52.0 From 2f243f26e5d07b9c0fc89a5aa7bd120cdf9af223 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Mon, 1 Jun 2020 19:04:39 +0200 Subject: [PATCH 037/228] readme update --- README.md | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0593bcb..3a73bec 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,6 @@ python demo.py --input_res 512 --arch resdcn_101 ctdet --demo /path/to/image/or/ python demo.py --input_res 512 --arch dla_34 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_dla_2x.pth --exp_wo --exp_wo_dim 512 ``` ### 4)Export weights for MobileNetSSD - To get the weights needed to run Mobilenet tests use [this](https://github.com/mive93/pytorch-ssd) fork of a Pytorch implementation of SSD network. ``` @@ -113,6 +112,34 @@ cd pytorch-ssd conda env create -f env_mobv2ssd.yml python run_ssd_live_demo.py mb2-ssd-lite ``` + +## Darknet Parser +tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*: +``` +// example of parsing yolo4 +tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names"); +net->print(); +``` +All models from darknet are now parsed directly from cfg, you still need to export the weights with the descripted tools in the previus section. +
+ Supported layers + convolutional + maxpool + avgpool + shortcut + upsample + route + reorg + region + yolo +
+
+ Supported activations + relu + leaky + mish +
+ ## Run the demo To run the an object detection demo follow these steps (example with yolov3): -- 2.52.0 From 0458f361b1589f9ea9c61bdbdeccf932a4b37ce1 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Mon, 1 Jun 2020 21:03:37 +0200 Subject: [PATCH 038/228] release layer wgs and version update --- include/tkDNN/Layer.h | 64 +++++++++++++++++++++++++++++++++++-------- include/tkDNN/tkdnn.h | 2 +- src/LayerWgs.cpp | 25 +++++------------ 3 files changed, 60 insertions(+), 31 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index e2295a7..9154e1b 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -108,24 +108,64 @@ public: // additional bias for DCN bool additional_bias; - dnnType *bias2_h, *bias2_d; + dnnType *bias2_h = nullptr, *bias2_d = nullptr; //batchnorm bool batchnorm; - dnnType *power_h; - dnnType *scales_h, *scales_d; - dnnType *mean_h, *mean_d; - dnnType *variance_h, *variance_d; + dnnType *power_h = nullptr; + dnnType *scales_h = nullptr, *scales_d = nullptr; + dnnType *mean_h = nullptr, *mean_d = nullptr; + dnnType *variance_h = nullptr, *variance_d = nullptr; //fp16 - __half *data16_h, *bias16_h; - __half *data16_d, *bias16_d; - __half *bias216_h, *bias216_d; + __half *data16_h = nullptr, *bias16_h = nullptr; + __half *data16_d = nullptr, *bias16_d = nullptr; + __half *bias216_h = nullptr, *bias216_d = nullptr; - __half *power16_h, *power16_d; - __half *scales16_h, *scales16_d; - __half *mean16_h, *mean16_d; - __half *variance16_h, *variance16_d; + __half *power16_h = nullptr; + __half *scales16_h = nullptr, *scales16_d = nullptr; + __half *mean16_h = nullptr, *mean16_d = nullptr; + __half *variance16_h = nullptr, *variance16_d = nullptr; + + void releaseHost(bool release32 = true, bool release16 = true) { + if(release32) { + if( data_h != nullptr) { delete [] data_h; data_h = nullptr; } + if( bias_h != nullptr) { delete [] bias_h; bias_h = nullptr; } + if( bias2_h != nullptr) { delete [] bias2_h; bias2_h = nullptr; } + if( scales_h != nullptr) { delete [] scales_h; scales_h = nullptr; } + if( mean_h != nullptr) { delete [] mean_h; mean_h = nullptr; } + if(variance_h != nullptr) { delete [] variance_h; variance_h = nullptr; } + if( power_h != nullptr) { delete [] power_h; power_h = nullptr; } + } + if(net->fp16 && release16) { + if( data16_h != nullptr) { delete [] data16_h; data16_h = nullptr; } + if( bias16_h != nullptr) { delete [] bias16_h; bias16_h = nullptr; } + if( bias216_h != nullptr) { delete [] bias216_h; bias216_h = nullptr; } + if( scales16_h != nullptr) { delete [] scales16_h; scales16_h = nullptr; } + if( mean16_h != nullptr) { delete [] mean16_h; mean16_h = nullptr; } + if(variance16_h != nullptr) { delete [] variance16_h; variance16_h = nullptr; } + if( power16_h != nullptr) { delete [] power16_h; power16_h = nullptr; } + + } + } + void releaseDevice(bool release32 = true, bool release16 = true) { + if(release32) { + if( data_d != nullptr) { cudaFree( data_d); data_d = nullptr; } + if( bias_d != nullptr) { cudaFree( bias_d); bias_d = nullptr; } + if( bias2_d != nullptr) { cudaFree( bias2_d); bias2_d = nullptr; } + if( scales_d != nullptr) { cudaFree( scales_d); scales_d = nullptr; } + if( mean_d != nullptr) { cudaFree( mean_d); mean_d = nullptr; } + if(variance_d != nullptr) { cudaFree(variance_d); variance_d = nullptr; } + } + if(net->fp16 && release16) { + if( data16_d != nullptr) { cudaFree( data16_d); data16_d = nullptr; } + if( bias16_d != nullptr) { cudaFree( bias16_d); bias16_d = nullptr; } + if( bias216_d != nullptr) { cudaFree( bias216_d); bias216_d = nullptr; } + if( scales16_d != nullptr) { cudaFree( scales16_d); scales16_d = nullptr; } + if( mean16_d != nullptr) { cudaFree( mean16_d); mean16_d = nullptr; } + if(variance16_d != nullptr) { cudaFree(variance16_d); variance16_d = nullptr; } + } + } }; diff --git a/include/tkDNN/tkdnn.h b/include/tkDNN/tkdnn.h index 554daa1..26aaa0b 100644 --- a/include/tkDNN/tkdnn.h +++ b/include/tkDNN/tkdnn.h @@ -5,4 +5,4 @@ #include "Layer.h" #include "NetworkRT.h" -#define TKDNN_VERSION 400 +#define TKDNN_VERSION 500 diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index 017563f..21edf79 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -80,7 +80,7 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, variance16_h = new __half[b_size]; scales16_h = new __half[b_size]; - cudaMalloc(&power16_d, b_size*sizeof(__half)); + //cudaMalloc(&power16_d, b_size*sizeof(__half)); cudaMalloc(&mean16_d, b_size*sizeof(__half)); cudaMalloc(&variance16_d, b_size*sizeof(__half)); cudaMalloc(&scales16_d, b_size*sizeof(__half)); @@ -91,11 +91,10 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, //init power array of ones cudaMemcpy(tmp_d, power_h, b_size*sizeof(float), cudaMemcpyHostToDevice); - float2half(tmp_d, power16_d, b_size); - cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); + //float2half(tmp_d, power16_d, b_size); + //cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); //mean array - cudaMemcpy(tmp_d, mean_h, b_size*sizeof(float), cudaMemcpyHostToDevice); float2half(tmp_d, mean16_d, b_size); cudaMemcpy(mean16_h, mean16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); @@ -109,24 +108,14 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, //conver scales float2half(scales_d, scales16_d, b_size); cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); + + cudaFree(tmp_d); } } LayerWgs::~LayerWgs() { - - delete [] data_h; - delete [] bias_h; - checkCuda( cudaFree(data_d) ); - checkCuda( cudaFree(bias_d) ); - - if(batchnorm) { - delete [] scales_h; - delete [] mean_h; - delete [] variance_h; - checkCuda( cudaFree(scales_d) ); - checkCuda( cudaFree(mean_d) ); - checkCuda( cudaFree(variance_d) ); - } + releaseHost(); + releaseDevice(); } }} -- 2.52.0 From 62e4a3f779053123f9f18370eb63f8b3595bc92a Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Tue, 2 Jun 2020 12:43:06 +0200 Subject: [PATCH 039/228] memory release --- include/tkDNN/Layer.h | 5 +++-- include/tkDNN/utils.h | 1 + src/Layer.cpp | 5 +++++ src/LayerWgs.cpp | 6 +++--- src/Network.cpp | 1 + src/NetworkRT.cpp | 1 + src/utils.cpp | 6 ++++++ 7 files changed, 20 insertions(+), 5 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 9154e1b..9bd8432 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -50,7 +50,7 @@ public: } void setFinal() { this->final = true; } dataDim_t input_dim, output_dim; - dnnType *dstData; //where results will be putted + dnnType *dstData = nullptr; //where results will be putted int id = 0; bool final; //if the layer is the final one @@ -122,7 +122,7 @@ public: __half *data16_d = nullptr, *bias16_d = nullptr; __half *bias216_h = nullptr, *bias216_d = nullptr; - __half *power16_h = nullptr; + __half *power16_h = nullptr, *power16_d = nullptr; __half *scales16_h = nullptr, *scales16_d = nullptr; __half *mean16_h = nullptr, *mean16_d = nullptr; __half *variance16_h = nullptr, *variance16_d = nullptr; @@ -164,6 +164,7 @@ public: if( scales16_d != nullptr) { cudaFree( scales16_d); scales16_d = nullptr; } if( mean16_d != nullptr) { cudaFree( mean16_d); mean16_d = nullptr; } if(variance16_d != nullptr) { cudaFree(variance16_d); variance16_d = nullptr; } + if( power16_d != nullptr) { cudaFree( power16_d); power16_d = nullptr; } } } }; diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index f9f6ae7..aa73e9e 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -116,5 +116,6 @@ void matrixMulAdd( cublasHandle_t handle, dnnType* srcData, dnnType* dstData, dnnType* add_vector, int dim, dnnType mul); void getMemUsage(double& vm_usage_kb, double& resident_set_kb); +void printCudaMemUsage(); void removePathAndExtension(const std::string &full_string, std::string &name); #endif //UTILS_H diff --git a/src/Layer.cpp b/src/Layer.cpp index 9f04ca5..a355b90 100644 --- a/src/Layer.cpp +++ b/src/Layer.cpp @@ -24,6 +24,11 @@ Layer::~Layer() { checkCUDNN( cudnnDestroyTensorDescriptor(srcTensorDesc) ); checkCUDNN( cudnnDestroyTensorDescriptor(dstTensorDesc) ); + + if(dstData != nullptr) { + cudaFree(dstData); + dstData = nullptr; + } } }} \ No newline at end of file diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index 21edf79..4afb7cc 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -80,7 +80,7 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, variance16_h = new __half[b_size]; scales16_h = new __half[b_size]; - //cudaMalloc(&power16_d, b_size*sizeof(__half)); + cudaMalloc(&power16_d, b_size*sizeof(__half)); cudaMalloc(&mean16_d, b_size*sizeof(__half)); cudaMalloc(&variance16_d, b_size*sizeof(__half)); cudaMalloc(&scales16_d, b_size*sizeof(__half)); @@ -91,8 +91,8 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, //init power array of ones cudaMemcpy(tmp_d, power_h, b_size*sizeof(float), cudaMemcpyHostToDevice); - //float2half(tmp_d, power16_d, b_size); - //cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); + float2half(tmp_d, power16_d, b_size); + cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); //mean array cudaMemcpy(tmp_d, mean_h, b_size*sizeof(float), cudaMemcpyHostToDevice); diff --git a/src/Network.cpp b/src/Network.cpp index 9adcc48..7fa291f 100644 --- a/src/Network.cpp +++ b/src/Network.cpp @@ -128,6 +128,7 @@ void Network::print() { } printCenteredTitle("", '=', 60); std::cout<<"\n"; + printCudaMemUsage(); } const char *Network::getNetworkRTName(const char *network_name){ networkName = network_name; diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 0824eba..9f53b06 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -134,6 +134,7 @@ NetworkRT::NetworkRT(Network *net, const char *name) { networkRT->markOutput(*input); std::cout<<"Selected maxBatchSize: "<getMaxBatchSize()<<"\n"; + printCudaMemUsage(); std::cout<<"Building tensorRT cuda engine...\n"; #if NV_TENSORRT_MAJOR >= 6 engineRT = builderRT->buildEngineWithConfig(*networkRT, *configRT); diff --git a/src/utils.cpp b/src/utils.cpp index 3bd9119..65030f0 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -197,6 +197,12 @@ void getMemUsage(double& vm_usage_kb, double& resident_set_kb){ resident_set_kb = rss * page_size_kb; } +void printCudaMemUsage() { + size_t free, total; + checkCuda( cudaMemGetInfo(&free, &total) ); + std::cout<<"GPU free memory: "< Date: Wed, 3 Jun 2020 11:01:17 +0200 Subject: [PATCH 040/228] Update README.md --- README.md | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3a73bec..fdfc877 100644 --- a/README.md +++ b/README.md @@ -177,24 +177,31 @@ N.b. Using FP16 inference will lead to some errors in the results (first or seco ### INT8 inference -To run the an object detection demo with INT8 inference follow these steps (example with yolov3): +To run the an object detection demo with INT8 inference three environment variables need to be set: + * ```export TKDNN_MODE=INT8```: set the 8-bit integer optimization + * ```export TKDNN_CALIB_IMG_PATH=/path/to/calibration/image_list.txt``` : image_list.txt has in each line the absolute path to a calibration image + * ```export TKDNN_CALIB_LABEL_PATH=/path/to/calibration/label_list.txt```: label_list.txt has in each line the absolute path to a calibration label + +You should provide image_list.txt and label_list.txt, using training images. However, if you want to quickly test the INT8 inference you can run (from this repo root folder) ``` -export TKDNN_MODE=INT8 # set the 8-bit integer optimization +bash scripts/download_validation.sh COCO +``` +to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD insted of COCO to download BDD validation. -# image_list.txt contains the list of the absolute paths to the calibration images -export TKDNN_CALIB_IMG_PATH=/path/to/calibration/image_list.txt - -# label_list.txt contains the list of the absolute paths to the calibration labels -export TKDNN_CALIB_LABEL_PATH=/path/to/calibration/label_list.txt +Then a complete example using yolo3 and COCO dataset would be: +``` +export TKDNN_MODE=INT8 +export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt +export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt rm yolo3_int8.rt # be sure to delete(or move) old tensorRT files ./test_yolo3 # run the yolo test (is slow) ./demo yolo3_int8.rt ../demo/yolo_test.mp4 y ``` -N.b. Using INT8 inference will lead to some errors in the results. - -N.b. The test will be slower: this is due to the INT8 calibration, which may take some time to complete. - -N.b. INT8 calibration requires TensorRT version greater than or equal to 6.0 +N.B. + * Using INT8 inference will lead to some errors in the results. + * The test will be slower: this is due to the INT8 calibration, which may take some time to complete. + * INT8 calibration requires TensorRT version greater than or equal to 6.0 + * Only 100 images are used to create the calibration table by default (set in the code). ### BatchSize bigger than 1 ``` -- 2.52.0 From 20303ac32ef8e28ca0ff94be633478d7755ae307 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Tue, 9 Jun 2020 20:53:05 +0200 Subject: [PATCH 041/228] cudnn8 compile --- src/LSTM.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 6ecf4fd..511fbee 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -86,7 +86,11 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig // RNN descriptors checkCUDNN(cudnnCreateRNNDescriptor(&rnnDesc)); - checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, +#if CUDNN_MAJOR > 7 + checkCUDNN(cudnnSetRNNDescriptor_v6(net->cudnnHandle, +#else + checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, +#endif rnnDesc, stateSize, numLayers, dropoutDesc, cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, //(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), -- 2.52.0 From be9e327aef7fb9c251ab31bca5f7a923b31395e2 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 9 Jun 2020 21:01:19 +0200 Subject: [PATCH 042/228] Fix minor, update readme Signed-off-by: Micaela Verucchi --- .gitignore | 2 +- README.md | 9 +++++++++ demo/demo/map.cpp | 4 ++-- scripts/install_OpenCV4.sh | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index d62c96d..b56526f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ build/ *.pk *.table demo/COCO_val2017 -demo/BDD100k_val \ No newline at end of file +demo/BDD100K_val \ No newline at end of file diff --git a/README.md b/README.md index fdfc877..1175465 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,18 @@ tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier and several discrete GPU. The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training. + +If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it or refer to https://hipert.unimore.it/ . + +``` Accepted paper @ IRC 2020, will soon been published. M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Time clustering and LiDAR-camera fusion on embedded platforms for self-driving cars", in proceedings in IEEE Robotic Computing (2020) +Accepted paper @ ETFA 2020, will soon been published. +M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capotondi, R. Cavicchioli, M. Bertogna, M. Solieri +"A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020) +``` + ## Index - [tkDNN](#tkdnn) - [Index](#index) diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 60aa7fa..d724db0 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -153,7 +153,7 @@ int main(int argc, char *argv[]) std::ofstream myfile; if(write_dets) - myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("000"))); + myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("labels/") + 7)); // save detections labels for(auto d:detected_bbox){ @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) f.det.push_back(b); if(write_dets) - myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; + myfile << d.cl << " "<< d.prob << " "<< b.x << " "<< b.y << " "<< b.w << " "<< b.h <<"\n"; if(show)// draw rectangle for detection cv::rectangle(batch_frames[0], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); diff --git a/scripts/install_OpenCV4.sh b/scripts/install_OpenCV4.sh index 8862cdc..f57c87b 100644 --- a/scripts/install_OpenCV4.sh +++ b/scripts/install_OpenCV4.sh @@ -62,5 +62,5 @@ make -j4 sudo make install sudo ldconfig -cd '~/Downloads/opencv4/lib/python3.6/site-packages' +cd ~/Downloads/opencv4/lib/python3.6/site-packages ln -s /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-aarch64-linux-gnu.so cv2.so -- 2.52.0 From c4e955eab54332eb8483629b0df42cd0ed94fae0 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 11 Jun 2020 12:43:34 +0200 Subject: [PATCH 043/228] Add different yolov4 size tests Signed-off-by: Micaela Verucchi --- include/tkDNN/utils.h | 2 +- src/utils.cpp | 16 +- tests/darknet/cfg/yolo4_320.cfg | 1156 +++++++++++++++++ .../darknet/cfg/{yolo4.cfg => yolo4_416.cfg} | 0 tests/darknet/cfg/yolo4_512.cfg | 1156 +++++++++++++++++ tests/darknet/cfg/yolo4_608.cfg | 1156 +++++++++++++++++ tests/darknet/{yolo4.cpp => yolo4_320.cpp} | 6 +- tests/darknet/yolo4_416.cpp | 34 + tests/darknet/yolo4_512.cpp | 34 + tests/darknet/yolo4_608.cpp | 34 + tests/test_rtinference/rtinference.cpp | 16 +- 11 files changed, 3596 insertions(+), 14 deletions(-) create mode 100644 tests/darknet/cfg/yolo4_320.cfg rename tests/darknet/cfg/{yolo4.cfg => yolo4_416.cfg} (100%) create mode 100644 tests/darknet/cfg/yolo4_512.cfg create mode 100644 tests/darknet/cfg/yolo4_608.cfg rename tests/darknet/{yolo4.cpp => yolo4_320.cpp} (85%) create mode 100644 tests/darknet/yolo4_416.cpp create mode 100644 tests/darknet/yolo4_512.cpp create mode 100644 tests/darknet/yolo4_608.cpp diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index aa73e9e..bca99f8 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -105,7 +105,7 @@ void printCenteredTitle(const char *title, char fill, int dim = 30); bool fileExist(const char *fname); void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string& test_folder, const std::string& weights_url); void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0); -int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10); +int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10, bool verbose=true); void printDeviceVector(int size, dnnType* vec_d, bool device = true); float getColor(const int c, const int x, const int max); void resize(int size, dnnType **data); diff --git a/src/utils.cpp b/src/utils.cpp index 65030f0..1ab57ad 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -83,7 +83,7 @@ void printDeviceVector(int size, dnnType* vec_d, bool device){ delete [] vec; } -int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int limit) { +int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int limit, bool verbose) { dnnType *data_h, *correct_h; const float eps = 0.02f; @@ -117,13 +117,15 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int delete [] correct_h; } - std::cout<<" | "; - if(diffs == 0) - std::cout< input_bins = { bin_path + "/layers/input.bin" }; @@ -15,9 +15,9 @@ int main() { bin_path + "/debug/layer161_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo4.cfg"; + std::string cfg_path = "../tests/darknet/cfg/yolo4_320.cfg"; std::string name_path = "../tests/darknet/names/coco.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/64PHAwrM6RCZbiR/download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); diff --git a/tests/darknet/yolo4_416.cpp b/tests/darknet/yolo4_416.cpp new file mode 100644 index 0000000..984b265 --- /dev/null +++ b/tests/darknet/yolo4_416.cpp @@ -0,0 +1,34 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4_416"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo4_416.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/982LxTQcNQfFQc4/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo4_512.cpp b/tests/darknet/yolo4_512.cpp new file mode 100644 index 0000000..414e9be --- /dev/null +++ b/tests/darknet/yolo4_512.cpp @@ -0,0 +1,34 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4_512"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo4_512.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/XN3FNXs3fnMaK5i/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo4_608.cpp b/tests/darknet/yolo4_608.cpp new file mode 100644 index 0000000..dda084f --- /dev/null +++ b/tests/darknet/yolo4_608.cpp @@ -0,0 +1,34 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4_608"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo4_608.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/Bg9r7kqDFJiFB4c/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} diff --git a/tests/test_rtinference/rtinference.cpp b/tests/test_rtinference/rtinference.cpp index a629168..e128ae8 100644 --- a/tests/test_rtinference/rtinference.cpp +++ b/tests/test_rtinference/rtinference.cpp @@ -1,4 +1,5 @@ #include +#include #include "tkdnn.h" #include /* srand, rand */ @@ -29,6 +30,7 @@ int main(int argc, char *argv[]) { int ret_tensorrt = 0; std::cout<<"Testing with batchsize: "< stats; printCenteredTitle(" TENSORRT inference ", '=', 30); float total_time = 0; for(int i=0; i<1200; i++) { @@ -46,18 +48,26 @@ int main(int argc, char *argv[]) { netRT.infer(dim, input_d); TKDNN_TSTOP total_time+= t_ns; + if(i> 1) + stats.push_back(t_ns); // control output - std::cout<<"Output Buffers: "< Date: Thu, 11 Jun 2020 16:09:01 +0200 Subject: [PATCH 044/228] Add script for inference FPS Signed-off-by: Micaela Verucchi --- scripts/test_inference.sh | 51 ++++++++++++++++++++++++++ tests/test_rtinference/rtinference.cpp | 25 +++++++++++-- 2 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 scripts/test_inference.sh diff --git a/scripts/test_inference.sh b/scripts/test_inference.sh new file mode 100644 index 0000000..fe8dac3 --- /dev/null +++ b/scripts/test_inference.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +function test_inference { + ./test_$1 + ./test_rtinference $1_$2.rt 1 + ./test_rtinference $1_$2.rt 4 +} + +sudo jeston_clock + +# modes=( 1 ) # only FP32 +# modes=( 1 2 ) # FP32 and FP16 +modes=( 1 2 3 ) # FP32, FP16 and INT8 + +rm times_rtinference.csv +for i in "${modes[@]}" +do + rm *rt + if [ $i -eq 1 ] + then + export TKDNN_MODE=FP32 + mode=fp32 + echo -e "${ORANGE}Test FP32${NC}" + fi + if [ $i -eq 2 ] + then + export TKDNN_MODE=FP16 + mode=fp16 + echo -e "${ORANGE}Test FP16${NC}" + fi + if [ $i -eq 3 ] + then + export TKDNN_MODE=INT8 + export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt + export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt + mode=int8 + echo -e "${ORANGE}Test INT8${NC}" + + fi + + export TKDNN_BATCHSIZE=4 + echo -e "${ORANGE}Batch $TKDNN_BATCHSIZE ${NC}" + + test_inference yolo4_320 $mode + test_inference yolo4_416 $mode + test_inference yolo4_512 $mode + test_inference yolo4_608 $mode +done + + + diff --git a/tests/test_rtinference/rtinference.cpp b/tests/test_rtinference/rtinference.cpp index e128ae8..76c2a33 100644 --- a/tests/test_rtinference/rtinference.cpp +++ b/tests/test_rtinference/rtinference.cpp @@ -18,6 +18,8 @@ int main(int argc, char *argv[]) { //convert network to tensorRT tk::dnn::NetworkRT netRT(NULL, argv[1]); + + tk::dnn::dataDim_t idim = netRT.input_dim; tk::dnn::dataDim_t odim = netRT.output_dim; @@ -63,11 +65,28 @@ int main(int argc, char *argv[]) { } } } - std::cout<<"Min: "<<*std::min_element(stats.begin(), stats.end())/BATCH_SIZE<<" ms\n"; - std::cout<<"Max: "<<*std::max_element(stats.begin(), stats.end())/BATCH_SIZE<<" ms\n"; + + double min = *std::min_element(stats.begin(), stats.end())/BATCH_SIZE; + double max = *std::max_element(stats.begin(), stats.end())/BATCH_SIZE; double mean =0; for(int i=0; i Date: Thu, 11 Jun 2020 20:54:17 +0200 Subject: [PATCH 045/228] New yolo4_512 download link Signed-off-by: Micaela Verucchi --- tests/darknet/yolo4_512.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/darknet/yolo4_512.cpp b/tests/darknet/yolo4_512.cpp index 414e9be..df3c2d0 100644 --- a/tests/darknet/yolo4_512.cpp +++ b/tests/darknet/yolo4_512.cpp @@ -17,7 +17,7 @@ int main() { std::string wgs_path = bin_path + "/layers"; std::string cfg_path = "../tests/darknet/cfg/yolo4_512.cfg"; std::string name_path = "../tests/darknet/names/coco.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/XN3FNXs3fnMaK5i/download"); + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/fjFDqFmiSARKxFe/download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); -- 2.52.0 From ab6d2d1766ea6761e79715a4c3c2eb60a19dc399 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Fri, 12 Jun 2020 11:39:46 +0200 Subject: [PATCH 046/228] Update README Signed-off-by: Micaela Verucchi --- README.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1175465..19b98a2 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,40 @@ The main goal of this project is to exploit NVIDIA boards as much as possible to If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it or refer to https://hipert.unimore.it/ . ``` -Accepted paper @ IRC 2020, will soon been published. +Accepted paper @ IRC 2020, will soon be published. M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Time clustering and LiDAR-camera fusion on embedded platforms for self-driving cars", in proceedings in IEEE Robotic Computing (2020) -Accepted paper @ ETFA 2020, will soon been published. +Accepted paper @ ETFA 2020, will soon be published. M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capotondi, R. Cavicchioli, M. Bertogna, M. Solieri "A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020) ``` +## Results +Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesion as the input size, on + * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); + * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 ); + * Tx2, Jetpack 4.2 (CUDA 10.0, CUDNN 7.3.1, tensorrt 5.0.6 ); + * Jetson Nano, Jetpack 4.4 (CUDA 10.2, CUDNN 8.0.0, tensorrt 7.1.0 ). + +| Platform | Network | FP32, B=1 | FP32, B=4 | FP16, B=1 | FP16, B=4 | INT8, B=1 | INT8, B=4 | +| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | +| RTX 2080Ti | yolo4 320 | 118,59 |237,31 | 207,81 | 443,32 | 262,37 | 530,93 | +| RTX 2080Ti | yolo4 416 | 104,81 |162,86 | 169,06 | 293,78 | 206,93 | 353,26 | +| RTX 2080Ti | yolo4 512 | 92,98 |132,43 | 140,36 | 215,17 | 165,35 | 254,96 | +| RTX 2080Ti | yolo4 608 | 63,77 |81,53 | 111,39 | 152,89 | 127,79 | 184,72 | +| AGX Xavier | yolo4 320 | 26,78 |32,05 | 57,14 | 79,05 | 73,15 | 97,56 | +| AGX Xavier | yolo4 416 | 19,96 |21,52 | 41,01 | 49,00 | 50,81 | 60,61 | +| AGX Xavier | yolo4 512 | 16,58 |16,98 | 31,12 | 33,84 | 37,82 | 41,28 | +| AGX Xavier | yolo4 608 | 9,45 |10,13 | 21,92 | 23,36 | 27,05 | 28,93 | +| Tx2 | yolo4 320 | 11,18 | 12,07 | 15,32 | 16,31 | - | - | +| Tx2 | yolo4 416 | 7,30 | 7,58 | 9,45 | 9,90 | - | - | +| Tx2 | yolo4 512 | 5,96 | 5,95 | 7,22 | 7,23 | - | - | +| Tx2 | yolo4 608 | 3,63 | 3,65 | 4,67 | 4,70 | - | - | +| Nano | yolo4 320 | 4,23 | 4,55 | 6,14 | 6,53 | - | - | +| Nano | yolo4 416 | 2,88 | 3,00 | 3,90 | 4,04 | - | - | +| Nano | yolo4 512 | 2,32 | 2,34 | 3,02 | 3,04 | - | - | +| Nano | yolo4 608 | 1,40 | 1,41 | 1,92 | 1,93 | - | - | + ## Index - [tkDNN](#tkdnn) - [Index](#index) -- 2.52.0 From 567dc0f75d5951eb677f95a14e8d3d9216d90634 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sun, 14 Jun 2020 12:57:29 +0200 Subject: [PATCH 047/228] cmake cudnn fix --- CMakeLists.txt | 2 + cmake/FindCUDNN.cmake | 91 +++++++++++++++++++++++++++++-------------- 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 376c175..4a64372 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,8 @@ SET(CUDA_SEPARABLE_COMPILATION ON) set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32) find_package(CUDNN REQUIRED) +include_directories(${CUDNN_INCLUDE_DIR}) + # compile file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu") diff --git a/cmake/FindCUDNN.cmake b/cmake/FindCUDNN.cmake index f240fcb..583b4a6 100644 --- a/cmake/FindCUDNN.cmake +++ b/cmake/FindCUDNN.cmake @@ -1,33 +1,66 @@ -# Find the header files +# find the library +if(CUDA_FOUND) + find_cuda_helper_libs(cudnn) + set(CUDNN_LIBRARY ${CUDA_cudnn_LIBRARY} CACHE FILEPATH "location of the cuDNN library") + unset(CUDA_cudnn_LIBRARY CACHE) -find_path(CUDNN_INCLUDE_DIR - ${CMAKE_SYSROOT}/usr/local/include - ${CMAKE_SYSROOT}/usr/include - /usr/local/nvidia/tensorrt/include/ - NO_DEFAULT_PATH -) + find_cuda_helper_libs(nvinfer) + set(NVINFER_LIBRARY ${CUDA_nvinfer_LIBRARY} CACHE FILEPATH "location of the nvinfer library") + unset(CUDA_nvinfer_LIBRARY CACHE) +endif() -set(OLD_ROOT ${CMAKE_FIND_ROOT_PATH}) -list(APPEND CMAKE_FIND_ROOT_PATH /) -list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.7) -list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.5) -find_library(CUDNN_LIB - NAMES cudnn - PATHS - /usr/local/driveworks/targets/${CMAKE_SYSTEM_PROCESSOR}-Linux/lib - /usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/ +# find the include +if(CUDNN_LIBRARY) + find_path(CUDNN_INCLUDE_DIR + cudnn.h + PATHS ${CUDA_TOOLKIT_INCLUDE} + DOC "location of cudnn.h" NO_DEFAULT_PATH -) -find_library(CUDNN_NVLIB - NAMES "nvinfer" - PATHS - /usr/local/driveworks/targets/${CMAKE_SYSTEM_PROCESSOR}-Linux/lib - /usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/ - NO_DEFAULT_PATH -) -set(CMAKE_FIND_ROOT_PATH ${OLD_ROOT}) + ) -set(CUDNN_LIBRARIES ${CUDNN_LIB} ${CUDNN_NVLIB}) -message("-- Found CUDNN: " ${CUDNN_LIB}) -message("-- Found NVINFER: " ${CUDNN_NVLIB}) -set(CUDNN_FOUND true) + if(NOT CUDNN_INCLUDE_DIR) + find_path(CUDNN_INCLUDE_DIR + cudnn.h + DOC "location of cudnn.h" + ) + endif() + + message("-- Found CUDNN: " ${CUDNN_LIBRARY}) + message("-- Found CUDNN include: " ${CUDNN_INCLUDE_DIR}) +endif() + +if(NVINFER_LIBRARY) + find_path(NVINFER_INCLUDE_DIR + NvInfer.h + PATHS ${CUDA_TOOLKIT_INCLUDE} + DOC "location of NvInfer.h" + NO_DEFAULT_PATH + ) + + if(NOT NVINFER_INCLUDE_DIR) + find_path(NVINFER_INCLUDE_DIR + NvInfer.h + DOC "location of NvInfer.h" + ) + endif() + + message("-- Found NVINFER: " ${NVINFER_LIBRARY}) + message("-- Found NVINFER include: " ${NVINFER_INCLUDE_DIR}) +endif() + + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(CUDNN + FOUND_VAR CUDNN_FOUND + REQUIRED_VARS + CUDNN_LIBRARY + CUDNN_INCLUDE_DIR + VERSION_VAR CUDNN_VERSION +) + +if(CUDNN_FOUND) + set(CUDNN_LIBRARIES ${CUDNN_LIBRARY} ${NVINFER_LIBRARY}) + set(CUDNN_INCLUDE_DIRS ${CUDNN_INCLUDE_DIR} ${NVINFER_INCLUDE_DIR}) +endif() + +set(CUDNN_FOUND true) \ No newline at end of file -- 2.52.0 From cbfc8ea4f2d8d52763b36e2c8ad2db4d4aa9d621 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sun, 14 Jun 2020 13:01:45 +0200 Subject: [PATCH 048/228] serialize fix --- src/NetworkRT.cpp | 2 +- tests/darknet/yolo3.cpp | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 9f53b06..6e86de1 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -595,7 +595,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, DeformConv2d *l) { bool NetworkRT::serialize(const char *filename) { - std::ofstream p(filename); + std::ofstream p(filename, std::ios::binary); if (!p) { FatalError("could not open plan output file"); return false; diff --git a/tests/darknet/yolo3.cpp b/tests/darknet/yolo3.cpp index ea53b84..464c3cf 100644 --- a/tests/darknet/yolo3.cpp +++ b/tests/darknet/yolo3.cpp @@ -5,19 +5,19 @@ #include "DarknetParser.h" int main() { - std::string bin_path = "yolo3"; + std::string bin_path = "yolov3-sppx"; std::vector input_bins = { bin_path + "/layers/input.bin" }; std::vector output_bins = { - bin_path + "/debug/layer82_out.bin", - bin_path + "/debug/layer94_out.bin", - bin_path + "/debug/layer106_out.bin" + bin_path + "/debug/layer89_out.bin", + bin_path + "/debug/layer101_out.bin", + bin_path + "/debug/layer113_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo3.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); + std::string cfg_path = "../tests/darknet/cfg/yolov3-sppx.cfg"; + std::string name_path = "../tests/darknet/names/coco4.names"; + //downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); -- 2.52.0 From 6dff675db7a9f58798535dfce8029b2dddac174b Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sun, 14 Jun 2020 13:03:48 +0200 Subject: [PATCH 049/228] fix wrong commit --- tests/darknet/yolo3.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/darknet/yolo3.cpp b/tests/darknet/yolo3.cpp index 464c3cf..e73d226 100644 --- a/tests/darknet/yolo3.cpp +++ b/tests/darknet/yolo3.cpp @@ -5,19 +5,19 @@ #include "DarknetParser.h" int main() { - std::string bin_path = "yolov3-sppx"; + std::string bin_path = "yolo3"; std::vector input_bins = { bin_path + "/layers/input.bin" }; std::vector output_bins = { - bin_path + "/debug/layer89_out.bin", - bin_path + "/debug/layer101_out.bin", - bin_path + "/debug/layer113_out.bin" + bin_path + "/debug/layer82_out.bin", + bin_path + "/debug/layer94_out.bin", + bin_path + "/debug/layer106_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolov3-sppx.cfg"; - std::string name_path = "../tests/darknet/names/coco4.names"; - //downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); + std::string cfg_path = "../tests/darknet/cfg/yolo3.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); @@ -31,4 +31,4 @@ int main() { delete net; delete netRT; return ret; -} +} \ No newline at end of file -- 2.52.0 From 285c77d6dd22fba207ec9c5fb5a43b3551e1e62a Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 15 Jun 2020 17:15:37 +0200 Subject: [PATCH 050/228] Imptove relative paths Signed-off-by: Micaela Verucchi Francesco Gatti --- CMakeLists.txt | 1 + tests/darknet/csresnext50-panet-spp.cpp | 4 ++-- tests/darknet/csresnext50-panet-spp_berkeley.cpp | 4 ++-- tests/darknet/yolo2.cpp | 4 ++-- tests/darknet/yolo2_voc.cpp | 4 ++-- tests/darknet/yolo2tiny.cpp | 4 ++-- tests/darknet/yolo3.cpp | 4 ++-- tests/darknet/yolo3_512.cpp | 4 ++-- tests/darknet/yolo3_berkeley.cpp | 4 ++-- tests/darknet/yolo3_coco4.cpp | 4 ++-- tests/darknet/yolo3_flir.cpp | 4 ++-- tests/darknet/yolo3tiny.cpp | 4 ++-- tests/darknet/yolo3tiny_512.cpp | 4 ++-- tests/darknet/yolo4.cpp | 4 ++-- tests/darknet/yolo4_berkeley.cpp | 4 ++-- 15 files changed, 29 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a64372..8c8619d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ if(DEBUG) add_definitions(-DDEBUG) endif() +add_definitions(-DTKDNN_PATH="${CMAKE_CURRENT_SOURCE_DIR}") #------------------------------------------------------------------------------- # CUDA diff --git a/tests/darknet/csresnext50-panet-spp.cpp b/tests/darknet/csresnext50-panet-spp.cpp index 1da95b2..a366e14 100644 --- a/tests/darknet/csresnext50-panet-spp.cpp +++ b/tests/darknet/csresnext50-panet-spp.cpp @@ -15,8 +15,8 @@ int main() { bin_path + "/debug/layer137_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/csresnext50-panet-spp.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/csresnext50-panet-spp.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download"); // parse darknet network diff --git a/tests/darknet/csresnext50-panet-spp_berkeley.cpp b/tests/darknet/csresnext50-panet-spp_berkeley.cpp index 47bbfd4..3cd0d52 100644 --- a/tests/darknet/csresnext50-panet-spp_berkeley.cpp +++ b/tests/darknet/csresnext50-panet-spp_berkeley.cpp @@ -15,8 +15,8 @@ int main() { bin_path + "/debug/layer137_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/csresnext50-panet-spp_berkeley.cfg"; - std::string name_path = "../tests/darknet/names/berkeley.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/csresnext50-panet-spp_berkeley.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/berkeley.names"; // FIXME: wrong weights // downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s//download"); diff --git a/tests/darknet/yolo2.cpp b/tests/darknet/yolo2.cpp index 7a46c31..978c137 100644 --- a/tests/darknet/yolo2.cpp +++ b/tests/darknet/yolo2.cpp @@ -13,8 +13,8 @@ int main() { bin_path + "/layers/output.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo2.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo2.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/nf4PJ3k8bxBETwL/download"); // parse darknet network diff --git a/tests/darknet/yolo2_voc.cpp b/tests/darknet/yolo2_voc.cpp index eab215b..94111e6 100644 --- a/tests/darknet/yolo2_voc.cpp +++ b/tests/darknet/yolo2_voc.cpp @@ -13,8 +13,8 @@ int main() { bin_path + "/layers/output.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo2_voc.cfg"; - std::string name_path = "../tests/darknet/names/voc.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo2_voc.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/voc.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/DJC5Fi2pEjfNDP9/download"); // parse darknet network diff --git a/tests/darknet/yolo2tiny.cpp b/tests/darknet/yolo2tiny.cpp index 64faa36..cc12109 100644 --- a/tests/darknet/yolo2tiny.cpp +++ b/tests/darknet/yolo2tiny.cpp @@ -13,8 +13,8 @@ int main() { bin_path + "/layers/output.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo2tiny.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo2tiny.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; // FIXME: wrong weights //downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s//download"); diff --git a/tests/darknet/yolo3.cpp b/tests/darknet/yolo3.cpp index e73d226..d9a684b 100644 --- a/tests/darknet/yolo3.cpp +++ b/tests/darknet/yolo3.cpp @@ -15,8 +15,8 @@ int main() { bin_path + "/debug/layer106_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo3.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); // parse darknet network diff --git a/tests/darknet/yolo3_512.cpp b/tests/darknet/yolo3_512.cpp index a67c3a7..6a5c20e 100644 --- a/tests/darknet/yolo3_512.cpp +++ b/tests/darknet/yolo3_512.cpp @@ -15,8 +15,8 @@ int main() { bin_path + "/debug/layer106_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo3_512.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3_512.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/RGecMeGLD4cXEWL/download"); // parse darknet network diff --git a/tests/darknet/yolo3_berkeley.cpp b/tests/darknet/yolo3_berkeley.cpp index a71fe83..016a8a2 100644 --- a/tests/darknet/yolo3_berkeley.cpp +++ b/tests/darknet/yolo3_berkeley.cpp @@ -15,8 +15,8 @@ int main() { bin_path + "/debug/layer106_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo3_berkeley.cfg"; - std::string name_path = "../tests/darknet/names/berkeley.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3_berkeley.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/berkeley.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/o5cHa4AjTKS64oD/download"); // parse darknet network diff --git a/tests/darknet/yolo3_coco4.cpp b/tests/darknet/yolo3_coco4.cpp index a651430..eaf9bd8 100644 --- a/tests/darknet/yolo3_coco4.cpp +++ b/tests/darknet/yolo3_coco4.cpp @@ -15,8 +15,8 @@ int main() { bin_path + "/debug/layer106_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo3_coco4.cfg"; - std::string name_path = "../tests/darknet/names/coco4.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3_coco4.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco4.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/o27NDzSAartbyc4/download"); // parse darknet network diff --git a/tests/darknet/yolo3_flir.cpp b/tests/darknet/yolo3_flir.cpp index 678f10f..24aac7f 100644 --- a/tests/darknet/yolo3_flir.cpp +++ b/tests/darknet/yolo3_flir.cpp @@ -15,8 +15,8 @@ int main() { bin_path + "/debug/layer106_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo3_flir.cfg"; - std::string name_path = "../tests/darknet/names/flir.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3_flir.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/flir.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/62DECncmF6bMMiH/download"); // parse darknet network diff --git a/tests/darknet/yolo3tiny.cpp b/tests/darknet/yolo3tiny.cpp index 01fc6f9..c33f7a8 100644 --- a/tests/darknet/yolo3tiny.cpp +++ b/tests/darknet/yolo3tiny.cpp @@ -14,8 +14,8 @@ int main() { bin_path + "/debug/layer23_out.bin", }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo3tiny.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3tiny.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/LMcSHtWaLeps8yN/download"); // parse darknet network diff --git a/tests/darknet/yolo3tiny_512.cpp b/tests/darknet/yolo3tiny_512.cpp index 8153b0d..ce4ce86 100644 --- a/tests/darknet/yolo3tiny_512.cpp +++ b/tests/darknet/yolo3tiny_512.cpp @@ -14,8 +14,8 @@ int main() { bin_path + "/debug/layer23_out.bin", }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo3tiny_512.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3tiny_512.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/8Zt6bHwHADqP4JC/download"); // parse darknet network diff --git a/tests/darknet/yolo4.cpp b/tests/darknet/yolo4.cpp index 80b12ce..65ac6ee 100644 --- a/tests/darknet/yolo4.cpp +++ b/tests/darknet/yolo4.cpp @@ -15,8 +15,8 @@ int main() { bin_path + "/debug/layer161_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo4.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download"); // parse darknet network diff --git a/tests/darknet/yolo4_berkeley.cpp b/tests/darknet/yolo4_berkeley.cpp index 8642eb1..89e9f04 100644 --- a/tests/darknet/yolo4_berkeley.cpp +++ b/tests/darknet/yolo4_berkeley.cpp @@ -15,8 +15,8 @@ int main() { bin_path + "/debug/layer161_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo4_berkeley.cfg"; - std::string name_path = "../tests/darknet/names/berkeley.names"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4_berkeley.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/berkeley.names"; downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download"); // parse darknet network -- 2.52.0 From 3d3a2427c9d17e6bc7ac83b56486568b1adba180 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Tue, 16 Jun 2020 12:36:45 +0200 Subject: [PATCH 051/228] viz yolo3 --- include/tkDNN/DarknetParser.h | 2 +- include/tkDNN/NetworkViz.h | 12 ++++++ include/tkDNN/utils.h | 4 ++ src/NetworkViz.cpp | 69 +++++++++++++++++++++++++++++++++++ tests/darknet/viz_yolo3.cpp | 54 +++++++++++++++++++++++++++ 5 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 include/tkDNN/NetworkViz.h create mode 100644 src/NetworkViz.cpp create mode 100644 tests/darknet/viz_yolo3.cpp diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index cbeed48..cd7fa48 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -1,6 +1,6 @@ #pragma once #include -#include "tkdnn.h" +#include "tkDNN/tkdnn.h" namespace tk { namespace dnn { diff --git a/include/tkDNN/NetworkViz.h b/include/tkDNN/NetworkViz.h new file mode 100644 index 0000000..c8b1bea --- /dev/null +++ b/include/tkDNN/NetworkViz.h @@ -0,0 +1,12 @@ +#pragma once +#include +#include +#include "tkdnn.h" + +namespace tk { namespace dnn { + +cv::Mat vizFloat2colorMap(cv::Mat map); +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim); +cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000); + +}} diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index aa73e9e..538a3f3 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -118,4 +118,8 @@ void matrixMulAdd( cublasHandle_t handle, dnnType* srcData, dnnType* dstData, void getMemUsage(double& vm_usage_kb, double& resident_set_kb); void printCudaMemUsage(); void removePathAndExtension(const std::string &full_string, std::string &name); +static inline bool isCudaPointer(void *data) { + cudaPointerAttributes attr; + return cudaPointerGetAttributes(&attr, data) == 0; +} #endif //UTILS_H diff --git a/src/NetworkViz.cpp b/src/NetworkViz.cpp new file mode 100644 index 0000000..6ac274c --- /dev/null +++ b/src/NetworkViz.cpp @@ -0,0 +1,69 @@ +#include +#include +#include +#include +#include "tkDNN/NetworkViz.h" + +namespace tk { namespace dnn { + +cv::Mat vizFloat2colorMap(cv::Mat map) { + + double min; + double max; + cv::minMaxIdx(map, &min, &max); + cv::Mat adjMap; + // expand your range to 0..255. Similar to histEq(); + map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); + //return adjMap; + + + cv::Mat falseColorsMap; + applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_HOT); + return falseColorsMap; +} + +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) { + dnnType *data = nullptr; + + // copy to CPU + if(isCudaPointer(dataInput)) { + data = new dnnType[dim.tot()]; + checkCuda( cudaMemcpy(data, dataInput, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); + } else { + data = dataInput; + } + + int gridDim = ceil(sqrt(dim.c)); + cv::Size gridSize(dim.w*gridDim, dim.h*gridDim); + cv::Mat grid = cv::Mat(gridSize, CV_8UC3, cv::Scalar(0)); + + for(int i=0; i= net->num_layers) + FatalError("Could not viz layer\n"); + return vizData2Mat(net->layers[layer]->dstData, net->layers[layer]->output_dim, imgdim); + + //cv::imwrite("viz/layer" + std::to_string(layer) + ".png", viz); + //cv::imshow("layer", viz); + //cv::waitKey(0); +} + +}} \ No newline at end of file diff --git a/tests/darknet/viz_yolo3.cpp b/tests/darknet/viz_yolo3.cpp new file mode 100644 index 0000000..374f7dd --- /dev/null +++ b/tests/darknet/viz_yolo3.cpp @@ -0,0 +1,54 @@ +#include +#include +#include + +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" +#include "NetworkViz.h" + +int main() { + std::string bin_path = "yolo3"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + // Load input and infer + dnnType *input_d; + dnnType *input_h; + readBinaryFile(input_bins[0], net->input_dim.tot(), &input_h, &input_d); + tk::dnn::dataDim_t dim = net->input_dim; + + dim.print(); + std::cout<<"infer\n"; + net->infer(dim, input_d); + + // output directory + std::string output_viz = "viz/"; + system( (std::string("mkdir -p ") + output_viz).c_str() ); + + for(int i=0; inum_layers; i++) { + std::string output_png = output_viz + "/layer" + std::to_string(i) + ".png"; + std::cout<<"saving "<releaseLayers(); + delete net; + return 0; +} + + \ No newline at end of file -- 2.52.0 From 1b8f45703f0f42a5a9bd1ca45fdda556345984c2 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Tue, 16 Jun 2020 12:41:48 +0200 Subject: [PATCH 052/228] darknet parser cpp --- include/tkDNN/DarknetParser.h | 272 ++-------------------------------- src/DarknetParser.cpp | 261 ++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+), 259 deletions(-) create mode 100644 src/DarknetParser.cpp diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index cd7fa48..f36469b 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -27,267 +27,21 @@ namespace tk { namespace dnn { std::vector layers; std::string activation = "linear"; + friend std::ostream& operator<<(std::ostream& os, const darknetFields_t& f){ + os << f.width << " " << f.height << " " << f.channels << " " << f.batch_normalize<< " " << f.filters << " " << f.activation<< " " << f.scale_xy; + return os; + } }; - std::ostream& operator<<(std::ostream& os, const darknetFields_t& f){ - os << f.width << " " << f.height << " " << f.channels << " " << f.batch_normalize<< " " << f.filters << " " << f.activation<< " " << f.scale_xy; - return os; - } - - std::string darknetParseType(const std::string& line){ - size_t start = line.find("["); - size_t end = line.find("]"); - if( start == std::string::npos || end == std::string::npos) - return ""; - start++; - std::string type = line.substr(start, end-start); - return type; - } - - bool divideNameAndValue(const std::string& line, std::string&name, std::string& value){ - size_t sep = line.find("="); - if(sep == std::string::npos) - return false; - - name = line.substr(0, sep); - value = line.substr(sep+1, line.size() - (sep+1)); - return true; - } - - std::vector fromStringToIntVec(const std::string& line, const char delimiter){ - std::stringstream linestream(line); - std::string value; - std::vector values; - - while(getline(linestream,value,delimiter)) - values.push_back(std::stoi(value)); - return values; - } - - bool darknetParseFields(const std::string& line, darknetFields_t& fields){ - - std::string name,value; - if(!divideNameAndValue(line, name, value)) - return false; - if(name.find("width") != std::string::npos) - fields.width = std::stoi(value); - else if(name.find("height") != std::string::npos) - fields.height = std::stoi(value); - else if(name.find("channels") != std::string::npos) - fields.channels = std::stoi(value); - else if(name.find("batch_normalize") != std::string::npos) - fields.batch_normalize = std::stoi(value); - else if(name.find("filters") != std::string::npos) - fields.filters = std::stoi(value); - else if(name.find("activation") != std::string::npos) - fields.activation = value; - else if(name.find("size") != std::string::npos){ - fields.size_x = std::stoi(value); - fields.size_y = std::stoi(value); - } - else if(name.find("size_x") != std::string::npos) - fields.size_x = std::stoi(value); - else if(name.find("size_y") != std::string::npos) - fields.size_y = std::stoi(value); - else if(name.find("stride") != std::string::npos){ - fields.stride_x = std::stoi(value); - fields.stride_y = std::stoi(value); - } - else if(name.find("stride_x") != std::string::npos) - fields.stride_x = std::stoi(value); - else if(name.find("stride_y") != std::string::npos) - fields.stride_y = std::stoi(value); - else if(name.find("pad") != std::string::npos) - fields.pad = std::stoi(value); - else if(name.find("classes") != std::string::npos) - fields.classes = std::stoi(value); - else if(name.find("num") != std::string::npos) - fields.num = std::stoi(value); - else if(name.find("coords") != std::string::npos) - fields.coords = std::stoi(value); - else if(name.find("groups") != std::string::npos) - fields.groups = std::stoi(value); - else if(name.find("scale_x_y") != std::string::npos) - fields.scale_xy = std::stof(value); - else if(name.find("from") != std::string::npos) - fields.layers.push_back(std::stof(value)); - else if(name.find("mask") != std::string::npos){ - auto vec = fromStringToIntVec(value, ','); - fields.n_mask = vec.size(); - } - else if(name.find("layers") != std::string::npos) - fields.layers = fromStringToIntVec(value, ','); - - else - std::cout<<"Not supported field: "< &netLayers, const std::vector& names) { - if(net == nullptr) - FatalError("Cant add a layer without a Net\n"); - - // padding compute - if(f.pad == 1) { - f.padding_x = f.padding_y = f.size_x /2; - } - //std::cout<<"Add layer: "<= netLayers.size()) FatalError("impossible to shortcut\n"); - //std::cout<<"shortcut to "<getLayerName()<<"\n"; - netLayers.push_back(new tk::dnn::Shortcut(net, netLayers[layerIdx])); - - } else if(f.type == "upsample") { - netLayers.push_back(new tk::dnn::Upsample(net, f.stride_x)); - - } else if(f.type == "route") { - if(f.layers.size() == 0) FatalError("no layers to Route\n"); - std::vector layers; - for(int i=0; i= netLayers.size()) FatalError("impossible to route\n"); - //std::cout<<"Route to "<getLayerName()<<"\n"; - layers.push_back(netLayers[layerIdx]); - } - netLayers.push_back(new tk::dnn::Route(net, layers.data(), layers.size())); - - } else if(f.type == "reorg") { - netLayers.push_back(new tk::dnn::Reorg(net, f.stride_x)); - - } else if(f.type == "region") { - netLayers.push_back(new tk::dnn::Region(net, f.classes, f.coords, f.num)); - - } else if(f.type == "yolo") { - std::string wgs = wgs_path + "/g" + std::to_string(netLayers.size()) + ".bin"; - //printf("%d %d %s %d %f\n", f.classes, f.num/f.n_mask, wgs.c_str(), f.n_mask, f.scale_xy); - tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy); - if(names.size() != f.classes) - FatalError("Mismatch between number of classes and names"); - l->classesNames = names; - netLayers.push_back(l); - - } else{ - FatalError("layer not supported: " + f.type); - } - - // add activation - if(netLayers.size() > 0 && f.activation != "linear") { - tkdnnActivationMode_t act; - if(f.activation == "relu") act = tkdnnActivationMode_t(CUDNN_ACTIVATION_RELU); - else if(f.activation == "leaky") act = tk::dnn::ACTIVATION_LEAKY; - else if(f.activation == "mish") act = tk::dnn::ACTIVATION_MISH; - else { FatalError("activation not supported: " + f.activation); } - netLayers[netLayers.size()-1] = new tk::dnn::Activation(net, act); - }; - } - - std::vector darknetReadNames(const std::string& names_file){ - std::ifstream if_names(names_file); - if(!if_names.is_open()) - FatalError("cloud not open names file: " + names_file); - - std::vector names; - std::string line; - while(std::getline(if_names, line)) - if(line != "") - names.push_back(line); - - if_names.close(); - return names; - } - - tk::dnn::Network* darknetParser(const std::string& cfg_file, const std::string& wgs_path, const std::string& names_file) { - - tk::dnn::Network *net = nullptr; - - // layers without activations to retrive correct id number - std::vector netLayers; - - std::ifstream if_cfg(cfg_file); - if(!if_cfg.is_open()) - FatalError("cloud not open cfg file: " + cfg_file); - - std::vector names = darknetReadNames(names_file); - - darknetFields_t fields; // will be filled with layers fields - std::string line; - while(std::getline(if_cfg, line)) { - // remove comments - std::size_t found = line.find("#"); - if ( found != std::string::npos ) { - line = line.substr(0, found); - } - - // skip empty lines - if(line.size() == 0) - continue; - - std::string type = darknetParseType(line); - if(type.size() > 0) { - // end of filled type - if(fields.type != "") { - if(fields.type == "net") - net = darknetAddNet(fields); - else - darknetAddLayer(net, fields, wgs_path, netLayers, names); - } - - // new type - //std::cout<<"type: "< fromStringToIntVec(const std::string& line, const char delimiter); + bool darknetParseFields(const std::string& line, darknetFields_t& fields); + tk::dnn::Network *darknetAddNet(darknetFields_t &fields); + void darknetAddLayer(tk::dnn::Network *net, darknetFields_t &f, std::string wgs_path, + std::vector &netLayers, const std::vector& names); + std::vector darknetReadNames(const std::string& names_file); + tk::dnn::Network* darknetParser(const std::string& cfg_file, const std::string& wgs_path, const std::string& names_file); }} diff --git a/src/DarknetParser.cpp b/src/DarknetParser.cpp new file mode 100644 index 0000000..5092595 --- /dev/null +++ b/src/DarknetParser.cpp @@ -0,0 +1,261 @@ +#include "tkDNN/DarknetParser.h" + +namespace tk { namespace dnn { + + std::string darknetParseType(const std::string& line){ + size_t start = line.find("["); + size_t end = line.find("]"); + if( start == std::string::npos || end == std::string::npos) + return ""; + start++; + std::string type = line.substr(start, end-start); + return type; + } + + bool divideNameAndValue(const std::string& line, std::string&name, std::string& value){ + size_t sep = line.find("="); + if(sep == std::string::npos) + return false; + + name = line.substr(0, sep); + value = line.substr(sep+1, line.size() - (sep+1)); + return true; + } + + std::vector fromStringToIntVec(const std::string& line, const char delimiter){ + std::stringstream linestream(line); + std::string value; + std::vector values; + + while(getline(linestream,value,delimiter)) + values.push_back(std::stoi(value)); + return values; + } + + bool darknetParseFields(const std::string& line, darknetFields_t& fields){ + + std::string name,value; + if(!divideNameAndValue(line, name, value)) + return false; + if(name.find("width") != std::string::npos) + fields.width = std::stoi(value); + else if(name.find("height") != std::string::npos) + fields.height = std::stoi(value); + else if(name.find("channels") != std::string::npos) + fields.channels = std::stoi(value); + else if(name.find("batch_normalize") != std::string::npos) + fields.batch_normalize = std::stoi(value); + else if(name.find("filters") != std::string::npos) + fields.filters = std::stoi(value); + else if(name.find("activation") != std::string::npos) + fields.activation = value; + else if(name.find("size") != std::string::npos){ + fields.size_x = std::stoi(value); + fields.size_y = std::stoi(value); + } + else if(name.find("size_x") != std::string::npos) + fields.size_x = std::stoi(value); + else if(name.find("size_y") != std::string::npos) + fields.size_y = std::stoi(value); + else if(name.find("stride") != std::string::npos){ + fields.stride_x = std::stoi(value); + fields.stride_y = std::stoi(value); + } + else if(name.find("stride_x") != std::string::npos) + fields.stride_x = std::stoi(value); + else if(name.find("stride_y") != std::string::npos) + fields.stride_y = std::stoi(value); + else if(name.find("pad") != std::string::npos) + fields.pad = std::stoi(value); + else if(name.find("classes") != std::string::npos) + fields.classes = std::stoi(value); + else if(name.find("num") != std::string::npos) + fields.num = std::stoi(value); + else if(name.find("coords") != std::string::npos) + fields.coords = std::stoi(value); + else if(name.find("groups") != std::string::npos) + fields.groups = std::stoi(value); + else if(name.find("scale_x_y") != std::string::npos) + fields.scale_xy = std::stof(value); + else if(name.find("from") != std::string::npos) + fields.layers.push_back(std::stof(value)); + else if(name.find("mask") != std::string::npos){ + auto vec = fromStringToIntVec(value, ','); + fields.n_mask = vec.size(); + } + else if(name.find("layers") != std::string::npos) + fields.layers = fromStringToIntVec(value, ','); + + else + std::cout<<"Not supported field: "< &netLayers, const std::vector& names) { + if(net == nullptr) + FatalError("Cant add a layer without a Net\n"); + + // padding compute + if(f.pad == 1) { + f.padding_x = f.padding_y = f.size_x /2; + } + //std::cout<<"Add layer: "<= netLayers.size()) FatalError("impossible to shortcut\n"); + //std::cout<<"shortcut to "<getLayerName()<<"\n"; + netLayers.push_back(new tk::dnn::Shortcut(net, netLayers[layerIdx])); + + } else if(f.type == "upsample") { + netLayers.push_back(new tk::dnn::Upsample(net, f.stride_x)); + + } else if(f.type == "route") { + if(f.layers.size() == 0) FatalError("no layers to Route\n"); + std::vector layers; + for(int i=0; i= netLayers.size()) FatalError("impossible to route\n"); + //std::cout<<"Route to "<getLayerName()<<"\n"; + layers.push_back(netLayers[layerIdx]); + } + netLayers.push_back(new tk::dnn::Route(net, layers.data(), layers.size())); + + } else if(f.type == "reorg") { + netLayers.push_back(new tk::dnn::Reorg(net, f.stride_x)); + + } else if(f.type == "region") { + netLayers.push_back(new tk::dnn::Region(net, f.classes, f.coords, f.num)); + + } else if(f.type == "yolo") { + std::string wgs = wgs_path + "/g" + std::to_string(netLayers.size()) + ".bin"; + //printf("%d %d %s %d %f\n", f.classes, f.num/f.n_mask, wgs.c_str(), f.n_mask, f.scale_xy); + tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy); + if(names.size() != f.classes) + FatalError("Mismatch between number of classes and names"); + l->classesNames = names; + netLayers.push_back(l); + + } else{ + FatalError("layer not supported: " + f.type); + } + + // add activation + if(netLayers.size() > 0 && f.activation != "linear") { + tkdnnActivationMode_t act; + if(f.activation == "relu") act = tkdnnActivationMode_t(CUDNN_ACTIVATION_RELU); + else if(f.activation == "leaky") act = tk::dnn::ACTIVATION_LEAKY; + else if(f.activation == "mish") act = tk::dnn::ACTIVATION_MISH; + else { FatalError("activation not supported: " + f.activation); } + netLayers[netLayers.size()-1] = new tk::dnn::Activation(net, act); + }; + } + + std::vector darknetReadNames(const std::string& names_file){ + std::ifstream if_names(names_file); + if(!if_names.is_open()) + FatalError("cloud not open names file: " + names_file); + + std::vector names; + std::string line; + while(std::getline(if_names, line)) + if(line != "") + names.push_back(line); + + if_names.close(); + return names; + } + + tk::dnn::Network* darknetParser(const std::string& cfg_file, const std::string& wgs_path, const std::string& names_file) { + + tk::dnn::Network *net = nullptr; + + // layers without activations to retrive correct id number + std::vector netLayers; + + std::ifstream if_cfg(cfg_file); + if(!if_cfg.is_open()) + FatalError("cloud not open cfg file: " + cfg_file); + + std::vector names = darknetReadNames(names_file); + + darknetFields_t fields; // will be filled with layers fields + std::string line; + while(std::getline(if_cfg, line)) { + // remove comments + std::size_t found = line.find("#"); + if ( found != std::string::npos ) { + line = line.substr(0, found); + } + + // skip empty lines + if(line.size() == 0) + continue; + + std::string type = darknetParseType(line); + if(type.size() > 0) { + // end of filled type + if(fields.type != "") { + if(fields.type == "net") + net = darknetAddNet(fields); + else + darknetAddLayer(net, fields, wgs_path, netLayers, names); + } + + // new type + //std::cout<<"type: "< Date: Tue, 16 Jun 2020 12:44:12 +0200 Subject: [PATCH 053/228] dealloc in test.h fix #36 --- include/tkDNN/test.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/include/tkDNN/test.h b/include/tkDNN/test.h index be3d891..4e238ff 100644 --- a/include/tkDNN/test.h +++ b/include/tkDNN/test.h @@ -1,7 +1,7 @@ #include int testInference(std::vector input_bins, std::vector output_bins, - tk::dnn::Network *net, tk::dnn::NetworkRT *netRT = nullptr) { + tk::dnn::Network *net, tk::dnn::NetworkRT *netRT = nullptr) { std::vector outputs; for(int i=0; inum_layers; i++) { @@ -67,7 +67,11 @@ int testInference(std::vector input_bins, std::vector std::cout<<"CUDNN vs TRT "; ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; } - } - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; - } \ No newline at end of file + delete [] out_h; + checkCuda( cudaFree(out) ); + } + delete [] input_h; + checkCuda( cudaFree(data) ); + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} \ No newline at end of file -- 2.52.0 From 1dfc69ba8916f913282c1fb68a117cdc72468f78 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Tue, 16 Jun 2020 13:19:30 +0200 Subject: [PATCH 054/228] viz yolo3 preprocess --- tests/darknet/viz_yolo3.cpp | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/darknet/viz_yolo3.cpp b/tests/darknet/viz_yolo3.cpp index 374f7dd..9e53116 100644 --- a/tests/darknet/viz_yolo3.cpp +++ b/tests/darknet/viz_yolo3.cpp @@ -1,32 +1,49 @@ #include #include #include +#include #include "tkdnn.h" #include "test.h" #include "DarknetParser.h" #include "NetworkViz.h" -int main() { +int main(int argc, char *argv[]) { + if(argc <2) + FatalError("you must provide an input image"); + std::string input_image = argv[1]; std::string bin_path = "yolo3"; - std::vector input_bins = { - bin_path + "/layers/input.bin" - }; std::string wgs_path = bin_path + "/layers"; std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3.cfg"; std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); + downloadWeightsifDoNotExist(wgs_path, bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); net->print(); - // Load input and infer + // input data dnnType *input_d; - dnnType *input_h; - readBinaryFile(input_bins[0], net->input_dim.tot(), &input_h, &input_d); - tk::dnn::dataDim_t dim = net->input_dim; + checkCuda( cudaMalloc(&input_d, sizeof(dnnType)*net->input_dim.tot())); + + // load image + cv::Mat frame, frameFloat; + frame = cv::imread(input_image); + cv::resize(frame, frame, cv::Size(net->input_dim.w, net->input_dim.h)); + frame.convertTo(frameFloat, CV_32FC3, 1/255.0); + + //split channels + cv::Mat bgr[3]; + cv::split(frameFloat,bgr);//split source + + //write channels + for(int i=0; iinput_dim.c; i++) { + int idx = i*frameFloat.rows*frameFloat.cols; + int ch = net->input_dim.c-1 -i; + checkCuda( cudaMemcpy(input_d + idx, (void*)bgr[ch].data, frameFloat.rows*frameFloat.cols*sizeof(dnnType), cudaMemcpyHostToDevice)); + } + tk::dnn::dataDim_t dim = net->input_dim; dim.print(); std::cout<<"infer\n"; net->infer(dim, input_d); @@ -44,7 +61,6 @@ int main() { //cv::waitKey(0); } - delete [] input_h; checkCuda(cudaFree(input_d)); net->releaseLayers(); delete net; -- 2.52.0 From 9f1e30eaa9c637e9ef30ab78e9f4809f4caac4e2 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Fri, 19 Jun 2020 18:55:42 +0200 Subject: [PATCH 055/228] Add shelfnet. Resnet18backbone works Signed-off-by: Micaela Verucchi --- CMakeLists.txt | 4 + src/kernels/activation_leaky.cu | 2 +- src/utils.cpp | 1 + tests/shelfnet/shelfnet.cpp | 213 ++++++++++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 tests/shelfnet/shelfnet.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c8619d..a2a8a06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,6 +103,10 @@ target_link_libraries(test_resnet101_cnet tkDNN) add_executable(test_dla34_cnet tests/centernet/dla34_cnet/dla34_cnet.cpp) target_link_libraries(test_dla34_cnet tkDNN) +# SHELFNET +add_executable(test_shelfnet tests/shelfnet/shelfnet.cpp) +target_link_libraries(test_shelfnet tkDNN) + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) diff --git a/src/kernels/activation_leaky.cu b/src/kernels/activation_leaky.cu index a9029ad..47dc18f 100644 --- a/src/kernels/activation_leaky.cu +++ b/src/kernels/activation_leaky.cu @@ -9,7 +9,7 @@ void activation_leaky(dnnType *input, dnnType *output, int size) { if (input[i]>0) output[i] = input[i]; else - output[i] = 0.1f*input[i]; + output[i] = 0.01f*input[i]; //FIME!! } } diff --git a/src/utils.cpp b/src/utils.cpp index 65030f0..87aad06 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -102,6 +102,7 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int } int diffs = 0; for(int i=0; i eps) { diffs += 1; diff --git a/tests/shelfnet/shelfnet.cpp b/tests/shelfnet/shelfnet.cpp new file mode 100644 index 0000000..3f8834a --- /dev/null +++ b/tests/shelfnet/shelfnet.cpp @@ -0,0 +1,213 @@ +#include +#include "tkdnn.h" + + +const char *output_bin1 = "shelfnet/debug/classification_headers-5.bin"; +const char *output_bin2 = "shelfnet/debug/regression_headers-5.bin"; +const char *input_bin = "shelfnet/debug/input.bin"; + +const char *backbone[] = { + "shelfnet/layers/backbone-conv1.bin", + "shelfnet/layers/backbone-layer1-0-conv1.bin", + "shelfnet/layers/backbone-layer1-0-conv2.bin", + "shelfnet/layers/backbone-layer1-1-conv1.bin", + "shelfnet/layers/backbone-layer1-1-conv2.bin", + "shelfnet/layers/backbone-layer2-0-conv1.bin", + "shelfnet/layers/backbone-layer2-0-conv2.bin", + "shelfnet/layers/backbone-layer2-0-downsample-0.bin", + "shelfnet/layers/backbone-layer2-1-conv1.bin", + "shelfnet/layers/backbone-layer2-1-conv2.bin", + "shelfnet/layers/backbone-layer3-0-conv1.bin", + "shelfnet/layers/backbone-layer3-0-conv2.bin", + "shelfnet/layers/backbone-layer3-0-downsample-0.bin", + "shelfnet/layers/backbone-layer3-1-conv1.bin", + "shelfnet/layers/backbone-layer3-1-conv2.bin", + "shelfnet/layers/backbone-layer4-0-conv1.bin", + "shelfnet/layers/backbone-layer4-0-conv2.bin", + "shelfnet/layers/backbone-layer4-0-downsample-0.bin", + "shelfnet/layers/backbone-layer4-1-conv1.bin", + "shelfnet/layers/backbone-layer4-1-conv2.bin"}; + +const char *conv_out[] = { + "shelfnet/layers/conv_out16-conv-conv.bin", + "shelfnet/layers/conv_out16-conv_out.bin", + "shelfnet/layers/conv_out32-conv-conv.bin", + "shelfnet/layers/conv_out32-conv_out.bin", + "shelfnet/layers/conv_out-conv-conv.bin", + "shelfnet/layers/conv_out-conv_out.bin"}; + +const char *decoder[] = { + "shelfnet/layers/decoder-bottom-conv1.bin", + "shelfnet/layers/decoder-up_conv_list-0-conv_atten.bin", + "shelfnet/layers/decoder-up_conv_list-0-conv-conv.bin", + "shelfnet/layers/decoder-up_conv_list-1-conv_atten.bin", + "shelfnet/layers/decoder-up_conv_list-1-conv-conv.bin", + "shelfnet/layers/decoder-up_dense_list-0-conv.bin", + "shelfnet/layers/decoder-up_dense_list-1-conv.bin"}; + + +const char *ladder[] = { + "shelfnet/layers/ladder-bottom-conv1.bin", + "shelfnet/layers/ladder-down_conv_list-0.bin", + "shelfnet/layers/ladder-down_conv_list-1.bin", + "shelfnet/layers/ladder-down_module_list-0-conv1.bin", + "shelfnet/layers/ladder-down_module_list-1-conv1.bin", + "shelfnet/layers/ladder-inconv-conv1.bin", + "shelfnet/layers/ladder-up_conv_list-0-conv_atten.bin", + "shelfnet/layers/ladder-up_conv_list-0-conv-conv.bin", + "shelfnet/layers/ladder-up_conv_list-1-conv_atten.bin", + "shelfnet/layers/ladder-up_conv_list-1-conv-conv.bin", + "shelfnet/layers/ladder-up_dense_list-0-conv.bin", + "shelfnet/layers/ladder-up_dense_list-1-conv.bin"}; + +const char *trans[] = { + "shelfnet/layers/trans1-conv.bin", + "shelfnet/layers/trans2-conv.bin", + "shelfnet/layers/trans3-conv.bin"}; +int main() +{ + + // downloadWeightsifDoNotExist(input_bin, "shelfnet", "https://cloud.hipert.unimore.it/s/x4ZfxBKN23zAJQp/download"); + + int classes = 19; + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1); + tk::dnn::Network net(dim); + + int bi = 0; + new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); + + + + for(int i=0; i<2; ++i){ + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + } + + std::vector features; + for(int i=0;i<3;++i){ + int out_channel = pow(2,7+i); + std::cout< Date: Mon, 22 Jun 2020 18:11:19 +0200 Subject: [PATCH 057/228] Shelfnet works on cuDNN. To test everything else Signed-off-by: Micaela Verucchi --- include/tkDNN/Layer.h | 18 ++- include/tkDNN/kernels.h | 2 +- include/tkDNN/pluginsRT/ShortcutRT.h | 12 +- src/LayerWgs.cpp | 3 +- src/NetworkRT.cpp | 4 +- src/Reshape.cpp | 5 + src/Resize.cpp | 38 ++++++ src/Shortcut.cpp | 15 ++- src/kernels/resize.cu | 41 +++---- src/kernels/shortcut.cu | 63 +++++++--- tests/shelfnet/shelfnet.cpp | 175 ++++++++++++++++++++++----- 11 files changed, 290 insertions(+), 86 deletions(-) create mode 100644 src/Resize.cpp diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 9bd8432..adbb5d1 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -21,6 +21,7 @@ enum layerType_t { LAYER_ACTIVATION_MISH, LAYER_FLATTEN, LAYER_RESHAPE, + LAYER_RESIZE, LAYER_MULADD, LAYER_POOLING, LAYER_SOFTMAX, @@ -70,6 +71,7 @@ public: case LAYER_ACTIVATION_MISH: return "ActivationMish"; case LAYER_FLATTEN: return "Flatten"; case LAYER_RESHAPE: return "Reshape"; + case LAYER_RESIZE: return "Resize"; case LAYER_MULADD: return "MulAdd"; case LAYER_POOLING: return "Pooling"; case LAYER_SOFTMAX: return "Softmax"; @@ -427,6 +429,19 @@ public: }; +/** + Resize layer +*/ +class Resize : public Layer { + +public: + Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed=false); + virtual ~Resize(); + virtual layerType_t getLayerType() { return LAYER_RESIZE; }; + + virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + +}; /** MulAdd layer @@ -545,7 +560,7 @@ public: class Shortcut : public Layer { public: - Shortcut(Network *net, Layer *backLayer); + Shortcut(Network *net, Layer *backLayer, bool mul=false); virtual ~Shortcut(); virtual layerType_t getLayerType() { return LAYER_SHORTCUT; }; @@ -553,6 +568,7 @@ public: public: Layer *backLayer; + bool mul = false; }; /** diff --git a/include/tkDNN/kernels.h b/include/tkDNN/kernels.h index 5d673c8..3d57132 100644 --- a/include/tkDNN/kernels.h +++ b/include/tkDNN/kernels.h @@ -24,7 +24,7 @@ void softmaxForward(float *input, int n, int batch, int batch_offset, int groups, int group_offset, int stride, float temp, float *output, cudaStream_t stream = cudaStream_t(0)); void shortcutForward(dnnType *srcData, dnnType *dstData, int n1, int c1, int h1, int w1, int s1, - int n2, int c2, int h2, int w2, int s2, + int n2, int c2, int h2, int w2, int s2, bool mul, cudaStream_t stream = cudaStream_t(0)); void upsampleForward(dnnType *srcData, dnnType *dstData, diff --git a/include/tkDNN/pluginsRT/ShortcutRT.h b/include/tkDNN/pluginsRT/ShortcutRT.h index 3eadd3f..e4a6a89 100644 --- a/include/tkDNN/pluginsRT/ShortcutRT.h +++ b/include/tkDNN/pluginsRT/ShortcutRT.h @@ -4,10 +4,11 @@ class ShortcutRT : public IPlugin { public: - ShortcutRT(tk::dnn::dataDim_t bdim) { + ShortcutRT(tk::dnn::dataDim_t bdim, bool mul) { this->bc = bdim.c; this->bh = bdim.h; this->bw = bdim.w; + this->mul = mul; } ~ShortcutRT(){ @@ -47,15 +48,14 @@ public: dnnType *dstData = reinterpret_cast(outputs[0]); checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream)); - for(int b=0; b < batchSize; ++b) - shortcutForward(srcDataBack + b*bc*bh*bw, dstData + b*c*h*w, 1, c, h, w, 1, 1, bc, bh, bw, 1, stream); + shortcutForward(srcDataBack, dstData, batchSize, c, h, w, 1, batchSize, bc, bh, bw, 1, mul, stream); return 0; } virtual size_t getSerializationSize() override { - return 6*sizeof(int); + return 6*sizeof(int) + sizeof(bool); } virtual void serialize(void* buffer) override { @@ -63,12 +63,14 @@ public: tk::dnn::writeBUF(buf, bc); tk::dnn::writeBUF(buf, bh); tk::dnn::writeBUF(buf, bw); + tk::dnn::writeBUF(buf, mul); tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); - + } int c, h, w; int bc, bh, bw; + bool mul; }; diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index 4afb7cc..820fb9e 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -26,10 +26,11 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, } readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek); + seek += outputs; this->batchnorm = batchnorm; if(batchnorm) { - seek += outputs; + readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek); seek += outputs; readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 6e86de1..5b75a46 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -512,7 +512,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) { else { // plugin version - IPlugin *plugin = new ShortcutRT(l->backLayer->output_dim); + IPlugin *plugin = new ShortcutRT(l->backLayer->output_dim, l->mul); ITensor **inputs = new ITensor*[2]; inputs[0] = input; inputs[1] = back_tens; @@ -682,7 +682,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa bdim.w = readBUF(buf); bdim.l = 1; - ShortcutRT *r = new ShortcutRT(bdim); + ShortcutRT *r = new ShortcutRT(bdim, readBUF(buf)); r->c = readBUF(buf); r->h = readBUF(buf); r->w = readBUF(buf); diff --git a/src/Reshape.cpp b/src/Reshape.cpp index c1d814a..f43c4ee 100644 --- a/src/Reshape.cpp +++ b/src/Reshape.cpp @@ -15,6 +15,11 @@ Reshape::Reshape(Network *net, dataDim_t new_dim) : Layer(net) { output_dim.w = new_dim.w; output_dim.l = new_dim.l; + output_dim = new_dim; + + if(input_dim.tot() != output_dim.tot()) + FatalError("Reshape dimension mismatch"); + } Reshape::~Reshape() { diff --git a/src/Resize.cpp b/src/Resize.cpp new file mode 100644 index 0000000..434d6b1 --- /dev/null +++ b/src/Resize.cpp @@ -0,0 +1,38 @@ +#include + +#include "Layer.h" +#include "kernels.h" + +namespace tk { namespace dnn { + +Resize::Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed) : Layer(net) { + + if(fixed){ + output_dim.c = scale_c; + output_dim.h = scale_h; + output_dim.w = scale_w; + } + else{ + output_dim.c *= scale_c; + output_dim.h *= scale_h; + output_dim.w *= scale_w; + } + + checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); +} + +Resize::~Resize() { + + checkCuda( cudaFree(dstData) ); +} + +dnnType* Resize::infer(dataDim_t &dim, dnnType* srcData) { + + resizeForward(srcData, dstData, dim.n, dim.c, dim.h, dim.w, + output_dim.c, output_dim.h, output_dim.w); + dim = output_dim; + + return dstData; +} + +}} \ No newline at end of file diff --git a/src/Shortcut.cpp b/src/Shortcut.cpp index 78a2f23..c9bddf6 100644 --- a/src/Shortcut.cpp +++ b/src/Shortcut.cpp @@ -5,15 +5,18 @@ namespace tk { namespace dnn { -Shortcut::Shortcut(Network *net, Layer *backLayer) : Layer(net) { +Shortcut::Shortcut(Network *net, Layer *backLayer, bool mul) : Layer(net) { this->backLayer = backLayer; + this->mul = mul; checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); - if( /*backLayer->output_dim.c != input_dim.c ||*/ - backLayer->output_dim.w != input_dim.w || - backLayer->output_dim.h != input_dim.h ) - FatalError("Shortcut dim missmatch"); + //FIXME + // if( /*backLayer->output_dim.c != input_dim.c ||*/ + // backLayer->output_dim.w != input_dim.w || + // backLayer->output_dim.h != input_dim.h ) + // FatalError("Shortcut dim missmatch"); + } Shortcut::~Shortcut() { @@ -26,7 +29,7 @@ dnnType* Shortcut::infer(dataDim_t &dim, dnnType* srcData) { dataDim_t bdim = this->backLayer->output_dim; checkCuda(cudaMemcpy(dstData, srcData, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); - shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1); + shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1, mul); //update data dimensions dim = output_dim; diff --git a/src/kernels/resize.cu b/src/kernels/resize.cu index 169b853..6059ef5 100644 --- a/src/kernels/resize.cu +++ b/src/kernels/resize.cu @@ -1,46 +1,33 @@ #include "kernels.h" #include -#define MIN(a,b) (((a)<(b))?(a):(b)) -#define MAX(a,b) (((a)>(b))?(a):(b)) -__global__ void resize_kernel( int i_N,float *x, int i_w, int i_h, int i_c, +__global__ void resize_kernel( int size,float *x, int i_w, int i_h, int i_c, int o_w, int o_h, int o_c, int batch, float *out) { - int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; - if(i >= i_N) return; + int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; + if(id >= size) return; - int out_index = i; - int out_w = i%o_w; - i = i/o_w; - int out_h = i%o_h; - i = i/o_h; - int out_c = i%o_c; - i = i/o_c; + int i = id % o_w; + id /= o_w; + int j = id % o_h; + id /= o_h; + int k = id % o_c; + id /= o_c; + int b = id % batch; - //copying last column/last row as padding - int in_index = ((i*i_c + MIN(out_c,i_c-1))*i_h + MIN(out_h,i_h-1))*i_w + MIN(out_w, i_w-1); - out[out_index] = x[in_index]; + int out_index = i + o_w*(j + o_h*(k + o_c*b)); + int add_index = i/(o_w/i_w) + i_w*(j/(o_h/i_h) + i_h*(k + i_c*b)); + out[out_index] = x[add_index]; } void resizeForward( dnnType* srcData, dnnType* dstData, int n, int i_c, int i_h, int i_w, int o_c, int o_h, int o_w, cudaStream_t stream ) { - int i_size = n*i_c*i_h*i_w; int o_size = n*o_c*o_h*o_w; int blocks = (o_size+255)/256; int threads = 256; - if(i_c == o_c && i_h == o_h && i_w == o_w ) - { - checkCuda(cudaMemcpy(dstData, srcData, i_size*sizeof(dnnType), cudaMemcpyDeviceToDevice)); - } - else - { - checkCuda(cudaMemset(dstData, 0, o_size*sizeof(dnnType))); - resize_kernel<<>>(o_size, srcData, i_w, i_h, i_c, o_w, o_h, o_c, n, dstData); - // printDeviceVector(i_size, srcData); - // printDeviceVector(o_size, dstData); - } + resize_kernel<<>>(o_size, srcData, i_w, i_h, i_c, o_w, o_h, o_c, n, dstData); } diff --git a/src/kernels/shortcut.cu b/src/kernels/shortcut.cu index 83539a8..c2d99bd 100644 --- a/src/kernels/shortcut.cu +++ b/src/kernels/shortcut.cu @@ -21,27 +21,60 @@ __global__ void shortcut_kernel(int size, int minw, int minh, int minc, int stri //out[out_index] += add[add_index]; } +__global__ void shortcut_mul_kernel(int size, int minw, int minh, int minc, int sample, int batch, + int w1, int h1, int c1, dnnType *mul, + int w2, int h2, int c2, float s1, float s2, dnnType *out) +{ + int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; + if (id >= size) return; + int i = id % minw; + id /= minw; + int j = id % minh; + id /= minh; + int k = id % minc; + id /= minc; + int b = id % batch; + + int out_index = i*sample + w1*(j*sample + h1*(k + c1*b)); + out[out_index] = out[out_index] * mul[k + c2*b]; +} + void shortcutForward(dnnType* srcData, dnnType* dstData, int n1, int c1, int h1, int w1, int s1, int n2, int c2, int h2, int w2, int s2, - cudaStream_t stream) + bool mul, cudaStream_t stream) { assert(n1 == n2); int batch = n1; - int minw = (w1 < w2) ? w1 : w2; - int minh = (h1 < h2) ? h1 : h2; - int minc = (c1 < c2) ? c1 : c2; + if(!mul){ + int minw = (w1 < w2) ? w1 : w2; + int minh = (h1 < h2) ? h1 : h2; + int minc = (c1 < c2) ? c1 : c2; + int stride = w1/w2; + int sample = w2/w1; + assert(stride == h1/h2); + assert(sample == h2/h1); + if(stride < 1) stride = 1; + if(sample < 1) sample = 1; - int stride = w1/w2; - int sample = w2/w1; - assert(stride == h1/h2); - assert(sample == h2/h1); - if(stride < 1) stride = 1; - if(sample < 1) sample = 1; + int size = batch * minw * minh * minc; + int blocks = (size+255)/256; + int threads = 256; + + shortcut_kernel<<>>(size, minw, minh, minc, stride, sample, batch, + w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData); + } + else{ + int minw = w1; + int minh = h1; + int minc = c1; + int sample = 1; - int size = batch * minw * minh * minc; - int blocks = (size+255)/256; - int threads = 256; - shortcut_kernel<<>>(size, minw, minh, minc, stride, sample, batch, - w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData); + int size = batch * minw * minh * minc; + int blocks = (size+255)/256; + int threads = 256; + + shortcut_mul_kernel<<>>(size, minw, minh, minc, sample, batch, + w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData); + } } diff --git a/tests/shelfnet/shelfnet.cpp b/tests/shelfnet/shelfnet.cpp index 3f8834a..0fb62c1 100644 --- a/tests/shelfnet/shelfnet.cpp +++ b/tests/shelfnet/shelfnet.cpp @@ -1,5 +1,9 @@ #include +#include +#include + #include "tkdnn.h" +#include "NetworkViz.h" const char *output_bin1 = "shelfnet/debug/classification_headers-5.bin"; @@ -29,35 +33,49 @@ const char *backbone[] = { "shelfnet/layers/backbone-layer4-1-conv2.bin"}; const char *conv_out[] = { + "shelfnet/layers/conv_out-conv-conv.bin", + "shelfnet/layers/conv_out-conv_out.bin", "shelfnet/layers/conv_out16-conv-conv.bin", "shelfnet/layers/conv_out16-conv_out.bin", "shelfnet/layers/conv_out32-conv-conv.bin", - "shelfnet/layers/conv_out32-conv_out.bin", - "shelfnet/layers/conv_out-conv-conv.bin", - "shelfnet/layers/conv_out-conv_out.bin"}; + "shelfnet/layers/conv_out32-conv_out.bin" + }; const char *decoder[] = { "shelfnet/layers/decoder-bottom-conv1.bin", - "shelfnet/layers/decoder-up_conv_list-0-conv_atten.bin", + "shelfnet/layers/decoder-bottom-conv12.bin", "shelfnet/layers/decoder-up_conv_list-0-conv-conv.bin", - "shelfnet/layers/decoder-up_conv_list-1-conv_atten.bin", - "shelfnet/layers/decoder-up_conv_list-1-conv-conv.bin", + "shelfnet/layers/decoder-up_conv_list-0-conv_atten.bin", "shelfnet/layers/decoder-up_dense_list-0-conv.bin", - "shelfnet/layers/decoder-up_dense_list-1-conv.bin"}; + "shelfnet/layers/decoder-up_conv_list-1-conv-conv.bin", + "shelfnet/layers/decoder-up_conv_list-1-conv_atten.bin", + "shelfnet/layers/decoder-up_dense_list-1-conv.bin" + }; const char *ladder[] = { - "shelfnet/layers/ladder-bottom-conv1.bin", - "shelfnet/layers/ladder-down_conv_list-0.bin", - "shelfnet/layers/ladder-down_conv_list-1.bin", - "shelfnet/layers/ladder-down_module_list-0-conv1.bin", - "shelfnet/layers/ladder-down_module_list-1-conv1.bin", "shelfnet/layers/ladder-inconv-conv1.bin", - "shelfnet/layers/ladder-up_conv_list-0-conv_atten.bin", + "shelfnet/layers/ladder-inconv-conv12.bin", + "shelfnet/layers/ladder-down_module_list-0-conv1.bin", + "shelfnet/layers/ladder-down_module_list-0-conv12.bin", + "shelfnet/layers/ladder-down_conv_list-0.bin", + + "shelfnet/layers/ladder-down_module_list-1-conv1.bin", + "shelfnet/layers/ladder-down_module_list-1-conv12.bin", + "shelfnet/layers/ladder-down_conv_list-1.bin", + + "shelfnet/layers/ladder-bottom-conv1.bin", + "shelfnet/layers/ladder-bottom-conv12.bin", + + + "shelfnet/layers/ladder-up_conv_list-0-conv-conv.bin", - "shelfnet/layers/ladder-up_conv_list-1-conv_atten.bin", - "shelfnet/layers/ladder-up_conv_list-1-conv-conv.bin", + "shelfnet/layers/ladder-up_conv_list-0-conv_atten.bin", "shelfnet/layers/ladder-up_dense_list-0-conv.bin", + + + "shelfnet/layers/ladder-up_conv_list-1-conv-conv.bin", + "shelfnet/layers/ladder-up_conv_list-1-conv_atten.bin", "shelfnet/layers/ladder-up_dense_list-1-conv.bin"}; const char *trans[] = { @@ -71,11 +89,11 @@ int main() int classes = 19; - // Network layout + // Network layout tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1); tk::dnn::Network net(dim); - int bi = 0; + int bi = 0, di = 0, li = 0, ci = 0; new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true); new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); @@ -105,24 +123,122 @@ int main() new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true); new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true); - - if(i != 2) - {new tk::dnn::Shortcut(&net, last); + new tk::dnn::Shortcut(&net, last); last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); - features.push_back(last);} + features.push_back(last); } - // for(int i=0; i up_out; + //bottom + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + std::cout<output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, features[1-i]); + + //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + up_out.push_back(last); + } + + //LADDER + + std::vector down_out; + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + + for(int i=0; i<2;++i){ + int out_channel = pow(2,6+i); + tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]); + + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, l_last); + l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + down_out.push_back(l_last); + + new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + } + + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.clear(); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + + new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, down_out[1-i]); + + // //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + up_out.push_back(last); + } + + + // for(int i=2;i>=0;--i){ + // new tk::dnn::Route(&net, &up_out[i], 1); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Conv2d (&net, 19, 3, 3, 1, 1, 1, 1, conv_out[ci++], false); + /*up_out[i] =*/ new tk::dnn::Resize(&net, 19, net.input_dim.h, net.input_dim.w, true); // } + new tk::dnn::Softmax(&net); - - const char *output_bin = "shelfnet/debug/backbone-layer4-1-bn2.bin"; + const char *output_bin = "shelfnet/debug/fofmaf.bin"; @@ -210,4 +326,7 @@ int main() // ret_cudnn_tensorrt |= checkResult(loc->output_dim.tot(), loc->dstData, rt_out4) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; // return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; + + cv::Mat viz = vizLayer2Mat(&net, net.num_layers-1); + cv::imwrite("test.png", viz); } -- 2.52.0 From 94e558003d1b87b751060d606d3137456a1eb7b1 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 23 Jun 2020 12:50:24 +0200 Subject: [PATCH 058/228] Shelfnet works on tensorRT (shortcut need to be fixed) Signed-off-by: Micaela Verucchi --- include/tkDNN/Layer.h | 3 +- include/tkDNN/NetworkRT.h | 1 + include/tkDNN/kernels.h | 2 +- include/tkDNN/pluginsRT/ActivationLeakyRT.h | 11 +- src/Activation.cpp | 10 +- src/NetworkRT.cpp | 21 +++- src/Shortcut.cpp | 8 +- src/kernels/activation_leaky.cu | 8 +- tests/shelfnet/shelfnet.cpp | 111 +++++++------------- 9 files changed, 75 insertions(+), 100 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index adbb5d1..7ea5e6c 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -225,8 +225,9 @@ class Activation : public Layer { public: int act_mode; float ceiling; + float slope; - Activation(Network *net, int act_mode, const float ceiling=0.0); + Activation(Network *net, int act_mode, const float ceiling=0.0, const float slope=0.1); virtual ~Activation(); virtual layerType_t getLayerType() { if(act_mode == CUDNN_ACTIVATION_CLIPPED_RELU) diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index ee1f728..3bd53d8 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -105,6 +105,7 @@ public: nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Flatten *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reshape *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Resize *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l); diff --git a/include/tkDNN/kernels.h b/include/tkDNN/kernels.h index 3d57132..d809129 100644 --- a/include/tkDNN/kernels.h +++ b/include/tkDNN/kernels.h @@ -4,7 +4,7 @@ #include "utils.h" void activationELUForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0)); -void activationLEAKYForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0)); +void activationLEAKYForward(dnnType *srcData, dnnType *dstData, int size, float slope, cudaStream_t stream = cudaStream_t(0)); void activationReLUCeilingForward(dnnType *srcData, dnnType *dstData, int size, const float ceiling, cudaStream_t stream = cudaStream_t(0)); void activationLOGISTICForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0)); void activationSIGMOIDForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0)); diff --git a/include/tkDNN/pluginsRT/ActivationLeakyRT.h b/include/tkDNN/pluginsRT/ActivationLeakyRT.h index d3f66fb..30bed7e 100644 --- a/include/tkDNN/pluginsRT/ActivationLeakyRT.h +++ b/include/tkDNN/pluginsRT/ActivationLeakyRT.h @@ -4,9 +4,8 @@ class ActivationLeakyRT : public IPlugin { public: - ActivationLeakyRT() { - - + ActivationLeakyRT(float s) { + slope = s; } ~ActivationLeakyRT(){ @@ -42,19 +41,21 @@ public: virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override { activationLEAKYForward((dnnType*)reinterpret_cast(inputs[0]), - reinterpret_cast(outputs[0]), batchSize*size, stream); + reinterpret_cast(outputs[0]), batchSize*size, slope, stream); return 0; } virtual size_t getSerializationSize() override { - return 1*sizeof(int); + return 1*sizeof(int) + 1*sizeof(float); } virtual void serialize(void* buffer) override { char *buf = reinterpret_cast(buffer); + tk::dnn::writeBUF(buf, slope); tk::dnn::writeBUF(buf, size); } int size; + float slope; }; diff --git a/src/Activation.cpp b/src/Activation.cpp index 28c7624..c97a717 100644 --- a/src/Activation.cpp +++ b/src/Activation.cpp @@ -5,11 +5,12 @@ namespace tk { namespace dnn { -Activation::Activation(Network *net, int act_mode, const float ceiling) : +Activation::Activation(Network *net, int act_mode, const float ceiling, const float slope) : Layer(net) { - this->act_mode = act_mode; - this->ceiling = ceiling; + this->act_mode = act_mode; + this->ceiling = ceiling; + this->slope = slope; checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) ); if(int(act_mode) < 100) { @@ -46,8 +47,7 @@ Activation::~Activation() { dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) { if(act_mode == ACTIVATION_LEAKY) { - activationLEAKYForward(srcData, dstData, dim.tot()); - + activationLEAKYForward(srcData, dstData, dim.tot(), this->slope); } else if(act_mode == ACTIVATION_MISH) { activationMishForward(srcData, dstData, dim.tot()); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 5b75a46..71662d3 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -236,6 +236,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) { return convert_layer(input, (Flatten*) l); if(type == LAYER_RESHAPE) return convert_layer(input, (Reshape*) l); + if(type == LAYER_RESIZE) + return convert_layer(input, (Resize*) l); if(type == LAYER_REORG) return convert_layer(input, (Reorg*) l); if(type == LAYER_REGION) @@ -389,13 +391,13 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { #if NV_TENSORRT_MAJOR < 6 // plugin version - IPlugin *plugin = new ActivationLeakyRT(); + IPlugin *plugin = new ActivationLeakyRT(l->slope); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); return lRT; #else IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kLEAKY_RELU); - lRT->setAlpha(0.1); + lRT->setAlpha(l->slope); checkNULL(lRT); return lRT; #endif @@ -469,13 +471,22 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Flatten *l) { ILayer* NetworkRT::convert_layer(ITensor *input, Reshape *l) { // std::cout<<"convert Reshape\n"; - l->output_dim.print(); IPlugin *plugin = new ReshapeRT(l->output_dim); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); return lRT; } +ILayer* NetworkRT::convert_layer(ITensor *input, Resize *l) { + // std::cout<<"convert Resize\n"; + + IResizeLayer *lRT = networkRT->addResize(*input); //default is kNEAREST + checkNULL(lRT); + Dims d{}; + lRT->setOutputDimensions(DimsCHW{l->output_dim.c, l->output_dim.h, l->output_dim.w}); + return lRT; +} + ILayer* NetworkRT::convert_layer(ITensor *input, Reorg *l) { //std::cout<<"convert Reorg\n"; @@ -503,7 +514,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) { ITensor *back_tens = tensors[l->backLayer]; - if(l->backLayer->output_dim.c == l->output_dim.c) + if(false) //l->backLayer->output_dim.c == l->output_dim.c && !l->mul) FIXME { IElementWiseLayer *lRT = networkRT->addElementWise(*input, *back_tens, ElementWiseOperation::kSUM); checkNULL(lRT); @@ -641,7 +652,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa //std::cout<(buf)); a->size = readBUF(buf); return a; } diff --git a/src/Shortcut.cpp b/src/Shortcut.cpp index c9bddf6..b1053c8 100644 --- a/src/Shortcut.cpp +++ b/src/Shortcut.cpp @@ -11,11 +11,9 @@ Shortcut::Shortcut(Network *net, Layer *backLayer, bool mul) : Layer(net) { this->mul = mul; checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); - //FIXME - // if( /*backLayer->output_dim.c != input_dim.c ||*/ - // backLayer->output_dim.w != input_dim.w || - // backLayer->output_dim.h != input_dim.h ) - // FatalError("Shortcut dim missmatch"); + if( ( backLayer->output_dim.c != input_dim.c && mul ) || + (( backLayer->output_dim.w != input_dim.w || backLayer->output_dim.h != input_dim.h ) && !mul ) ) + FatalError("Shortcut dim missmatch"); } diff --git a/src/kernels/activation_leaky.cu b/src/kernels/activation_leaky.cu index 47dc18f..2ecb3af 100644 --- a/src/kernels/activation_leaky.cu +++ b/src/kernels/activation_leaky.cu @@ -1,7 +1,7 @@ #include "kernels.h" __global__ -void activation_leaky(dnnType *input, dnnType *output, int size) { +void activation_leaky(dnnType *input, dnnType *output, int size, float slope) { int i = blockDim.x*blockIdx.x + threadIdx.x; @@ -9,7 +9,7 @@ void activation_leaky(dnnType *input, dnnType *output, int size) { if (input[i]>0) output[i] = input[i]; else - output[i] = 0.01f*input[i]; //FIME!! + output[i] = slope*input[i]; } } @@ -17,12 +17,12 @@ void activation_leaky(dnnType *input, dnnType *output, int size) { /** ELU activation function */ -void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream) +void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, float slope, cudaStream_t stream) { int blocks = (size+255)/256; int threads = 256; - activation_leaky<<>>(srcData, dstData, size); + activation_leaky<<>>(srcData, dstData, size, slope); } diff --git a/tests/shelfnet/shelfnet.cpp b/tests/shelfnet/shelfnet.cpp index 0fb62c1..ab28d90 100644 --- a/tests/shelfnet/shelfnet.cpp +++ b/tests/shelfnet/shelfnet.cpp @@ -6,8 +6,6 @@ #include "NetworkViz.h" -const char *output_bin1 = "shelfnet/debug/classification_headers-5.bin"; -const char *output_bin2 = "shelfnet/debug/regression_headers-5.bin"; const char *input_bin = "shelfnet/debug/input.bin"; const char *backbone[] = { @@ -95,14 +93,14 @@ int main() int bi = 0, di = 0, li = 0, ci = 0; new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true); - new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); for(int i=0; i<2; ++i){ new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); - new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); new tk::dnn::Shortcut(&net, last); last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); @@ -113,7 +111,7 @@ int main() int out_channel = pow(2,7+i); std::cout< up_out; //bottom new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); - new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); new tk::dnn::Shortcut(&net, last); last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); @@ -153,7 +151,7 @@ int main() //up-conv std::cout<output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true); @@ -168,7 +166,7 @@ int main() //up-dense new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true); - last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); up_out.push_back(last); } @@ -176,7 +174,7 @@ int main() std::vector down_out; new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); - new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); new tk::dnn::Shortcut(&net, last); new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); @@ -186,7 +184,7 @@ int main() tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]); new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); - new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); new tk::dnn::Shortcut(&net, l_last); l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); @@ -197,7 +195,7 @@ int main() } new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); - new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); new tk::dnn::Shortcut(&net, last); last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); @@ -208,7 +206,7 @@ int main() int out_channel = pow(2,7-i); //up-conv new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); - last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true); @@ -223,7 +221,7 @@ int main() // //up-dense new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); - last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); up_out.push_back(last); } @@ -231,29 +229,26 @@ int main() // for(int i=2;i>=0;--i){ // new tk::dnn::Route(&net, &up_out[i], 1); new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true); - new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); new tk::dnn::Conv2d (&net, 19, 3, 3, 1, 1, 1, 1, conv_out[ci++], false); - /*up_out[i] =*/ new tk::dnn::Resize(&net, 19, net.input_dim.h, net.input_dim.w, true); + // /*up_out[i] =*/ new tk::dnn::Resize(&net, 19, net.input_dim.h, net.input_dim.w, true); // } - new tk::dnn::Softmax(&net); + // new tk::dnn::Softmax(&net); - const char *output_bin = "shelfnet/debug/fofmaf.bin"; - + const char *output_bin = "shelfnet/debug/conv_out-conv_out.bin"; - // Load input dnnType *data; dnnType *input_h; readBinaryFile(input_bin, dim.tot(), &input_h, &data); std::cout<<"Input:"<output_dim; - // dnnType *cudnn_out2 = loc5[0]->dstData; - // tk::dnn::dataDim_t out_dim2 = loc5[0]->output_dim; + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TKDNN_TSTART + netRT.infer(dim2, data); + TKDNN_TSTOP + dim2.print(); + } - // tk::dnn::dataDim_t dim2 = dim; - // printCenteredTitle(" TENSORRT inference ", '=', 30); - // { - // dim2.print(); - // TKDNN_TSTART - // netRT.infer(dim2, data); - // TKDNN_TSTOP - // dim2.print(); - // } + dnnType *rt_out1 = (dnnType *)netRT.buffersRT[1]; - // dnnType *rt_out1 = (dnnType *)netRT.buffersRT[1]; - // dnnType *rt_out2 = (dnnType *)netRT.buffersRT[2]; - // dnnType *rt_out3 = (dnnType *)netRT.buffersRT[3]; - // dnnType *rt_out4 = (dnnType *)netRT.buffersRT[4]; - - printCenteredTitle(std::string(" RESNET CHECK RESULTS ").c_str(), '=', 30); + printCenteredTitle(std::string(" CHECK RESULTS ").c_str(), '=', 30); dnnType *out1, *out1_h; int odim1 = dim1.tot(); readBinaryFile(output_bin, odim1, &out1_h, &out1); - printDeviceVector(64, out1); - - // dnnType *out2, *out2_h; - // int odim2 = out_dim2.tot(); - // readBinaryFile(output_bin2, odim2, &out2_h, &out2); - // int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; - + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; std::cout << "CUDNN vs correct" << std::endl; - checkResult(odim1, cudnn_out, out1, true, 20) == 0 ? 0 : ERROR_CUDNN; + ret_cudnn |= checkResult(odim1, cudnn_out, out1, true, 20) == 0 ? 0 : ERROR_CUDNN; - // std::cout << "TRT vs correct" << std::endl; - // checkResult(odim1, rt_out1, out1) == 0 ? 0 : ERROR_TENSORRT; - // ret_tensorrt |= checkResult(odim2, rt_out2, out2) == 0 ? 0 : ERROR_TENSORRT; + std::cout << "TRT vs correct" << std::endl; + ret_tensorrt |=checkResult(odim1, rt_out1, out1) == 0 ? 0 : ERROR_TENSORRT; - // std::cout << "CUDNN vs TRT " << std::endl; - // ret_cudnn_tensorrt |= checkResult(odim1, cudnn_out1, rt_out1) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - // ret_cudnn_tensorrt |= checkResult(odim2, cudnn_out2, rt_out2) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - // std::cout << "---------------------------------------------------" << std::endl; - // std::cout << "Confidence CUDNN" << std::endl; - // printDeviceVector(64, conf->dstData, true); - // std::cout << "Locations CUDNN" << std::endl; - // printDeviceVector(64, loc->dstData, true); - // std::cout << "---------------------------------------------------" << std::endl; - - // std::cout << "Confidence tensorRT" << std::endl; - // printDeviceVector(64, rt_out3, true); - // std::cout << "Locations tensorRT" << std::endl; - // printDeviceVector(64, rt_out4, true); - // std::cout << "---------------------------------------------------" << std::endl; - - // std::cout << "CUDNN vs TRT " << std::endl; - // ret_cudnn_tensorrt |= checkResult(conf->output_dim.tot(), conf->dstData, rt_out3) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - // ret_cudnn_tensorrt |= checkResult(loc->output_dim.tot(), loc->dstData, rt_out4) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - - // return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; + std::cout << "CUDNN vs TRT " << std::endl; + ret_cudnn_tensorrt |= checkResult(odim1, cudnn_out, rt_out1) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; cv::Mat viz = vizLayer2Mat(&net, net.num_layers-1); cv::imwrite("test.png", viz); + + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; } -- 2.52.0 From 082920f3f56d66853816e785b58f9d0f0786273f Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 23 Jun 2020 20:01:47 +0200 Subject: [PATCH 059/228] Shelfnet works, also visualization. Postprocessing need to be parallelized Signed-off-by: Micaela Verucchi --- CMakeLists.txt | 3 + demo/demo/seg_demo.cpp | 112 ++++++++++++ include/tkDNN/Layer.h | 6 +- include/tkDNN/NetworkViz.h | 4 +- include/tkDNN/SegmentationNN.h | 225 +++++++++++++++++++++++++ include/tkDNN/utils.h | 2 +- scripts/test_all_tests.sh | 1 + src/NetworkRT.cpp | 3 +- src/NetworkViz.cpp | 15 +- src/Resize.cpp | 3 +- tests/shelfnet/shelfnet.cpp | 8 +- tests/test_rtinference/rtinference.cpp | 4 +- 12 files changed, 366 insertions(+), 20 deletions(-) create mode 100644 demo/demo/seg_demo.cpp create mode 100644 include/tkDNN/SegmentationNN.h diff --git a/CMakeLists.txt b/CMakeLists.txt index a2a8a06..88e94e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,6 +117,9 @@ target_link_libraries(map_demo tkDNN) add_executable(demo demo/demo/demo.cpp) target_link_libraries(demo tkDNN) +add_executable(seg_demo demo/demo/seg_demo.cpp) +target_link_libraries(seg_demo tkDNN) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/demo/demo/seg_demo.cpp b/demo/demo/seg_demo.cpp new file mode 100644 index 0000000..6f84b7d --- /dev/null +++ b/demo/demo/seg_demo.cpp @@ -0,0 +1,112 @@ +#include +#include +#include /* srand, rand */ +#include +#include + +#include "SegmentationNN.h" + +bool gRun; +bool SAVE_RESULT = false; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + std::cout<<"detection\n"; + signal(SIGINT, sig_handler); + + + std::string net = "shelfnet_fp32.rt"; + if(argc > 1) + net = argv[1]; + std::string input = "../../ShelfNet/ShelfNet18_realtime/data/leftImg8bit/test/modena/000302.png"; + if(argc > 2) + input = argv[2]; + int n_batch = 1; + if(argc > 3) + n_batch = atoi(argv[3]); + bool show = false; + if(argc > 4) + show = atoi(argv[4]); + + if(n_batch < 1 || n_batch > 64) + FatalError("Batch dim not supported"); + + if(!show) + SAVE_RESULT = true; + + int n_classes = 19; + + tk::dnn::SegmentationNN segNN; + segNN.init(net, n_classes, n_batch); + + gRun = true; + + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + + cv::Mat frame; + if(show) + cv::namedWindow("segmentation", cv::WINDOW_NORMAL); + + std::vector batch_frame; + std::vector batch_dnn_input; + + while(gRun) { + batch_dnn_input.clear(); + batch_frame.clear(); + + for(int bi=0; bi< n_batch; ++bi){ + cap >> frame; + if(!frame.data) + break; + + batch_frame.push_back(frame); + + // this will be resized to the net format + batch_dnn_input.push_back(frame.clone()); + } + if(!frame.data) + break; + + //inference + segNN.update(batch_dnn_input, n_batch); + segNN.draw(); + + if(show){ + for(int bi=0; bi< n_batch; ++bi){ + cv::imshow("segmentation", batch_frame[bi]); + cv::waitKey(1); + } + } + if(n_batch == 1 && SAVE_RESULT) + resultVideo << frame; + } + + std::cout<<"segmentation end\n"; + double mean = 0; + + std::cout< +#include +#include +#include +#include +#include "utils.h" + +#include +#include +#include +#include + + + +#include "tkdnn.h" +#include "NetworkViz.h" + +namespace tk { namespace dnn { + +class SegmentationNN { + + protected: + tk::dnn::NetworkRT *netRT = nullptr; + int nBatches = 1; + + std::vector originalSize; + std::vector masks; + cv::Mat bgr[3]; + dnnType *input; + dnnType *input_d; + float* confidences_h; + + /** + * This method preprocess the image, before feeding it to the NN. + * + * @param frame original frame to adapt for inference. + * @param bi batch index + */ + void preprocess(cv::Mat &frame, const int bi=0) { + + frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0); + + cv::split(frame, bgr); + float mean[] = {0.485, 0.456, 0.406}; + float stddev[] = {0.229, 0.224, 0.225}; + for(int i=0; i<3; i++){ + bgr[2-i] -= mean[i]; + bgr[2-i] /= stddev[i]; + } + cv::merge(bgr, 3, frame); + + int crop_size = netRT->input_dim.w; + int H = frame.rows; + int W = frame.cols; + cv::Mat frame_cropped; + + cv::Mat mask(frame.size(), CV_8UC3, cv::Scalar(255,255,255)); + + if(H != W){ + if(H < W){ + int top = (W - H)/2; + int bottom = W - top - H; + cv::copyMakeBorder(frame, frame_cropped, top, bottom, 0, 0, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) ); + cv::copyMakeBorder(mask, mask, top, bottom, 0, 0, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) ); + } + else{ + int left = (H - W)/2; + int right = H - left - W; + cv::copyMakeBorder(frame, frame_cropped, 0, 0, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) ); + cv::copyMakeBorder(mask, mask, 0, 0, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) ); + } + } + + resize(frame_cropped, frame_cropped, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); + resize(mask, mask, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); + masks[bi] = mask.clone(); + + cv::split(frame_cropped, bgr); + for (int i = 0; i < netRT->input_dim.c; i++){ + int idx = i * frame_cropped.rows * frame_cropped.cols; + int ch = netRT->input_dim.c-1 -i; + memcpy((void *)&input[idx + netRT->input_dim.tot()*bi], (void *)bgr[ch].data, frame_cropped.rows * frame_cropped.cols * sizeof(dnnType)); + } + checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); + } + + /** + * This method postprocess the output of the NN to obtain the correct + * boundig boxes. + * + * @param bi batch index + */ + void postprocess(const int bi=0) { + dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + + dataDim_t odim = netRT->output_dim; + + + checkCuda(cudaMemcpy(confidences_h, rt_out, odim.tot() * sizeof(float), cudaMemcpyDeviceToHost)); + + for(int i=0;i max_conf){ + max_conf = cur_conf; + max_id = k; + } + } + confidences_h[bi*odim.tot()+0*odim.h*odim.w+i*odim.h+j] = max_id; + } + } + dataDim_t vdim = odim; + vdim.c = 1; + segmented[bi] = vizData2Mat(confidences_h, vdim, 1024, 0, 18); + }; + + public: + int classes = 0; + std::vector stats; /*keeps track of inference times (ms)*/ + std::vector classesNames; + std::vector segmented; + + SegmentationNN() {}; + ~SegmentationNN(){}; + + /** + * Method used to inialize the class, allocate memory and compute + * needed data. + * + * @param tensor_path path to the rt file og the NN. + * @param n_classes number of classes for the given dataset. + * @param n_batches maximum number of batches to use in inference + * @return true if everything is correct, false otherwise. + */ + bool init(const std::string& tensor_path, const int n_classes=19, const int n_batches=1){ + std::cout<<(tensor_path).c_str()<<"\n"; + if(!fileExist(tensor_path.c_str())) + FatalError("This file do not exists" + tensor_path ); + + netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); + classes = n_classes; + nBatches = n_batches; + + checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + + confidences_h = (float *)malloc(netRT->output_dim.tot() * sizeof(float)); + + segmented.resize(nBatches); + masks.resize(nBatches); + } + + /** + * This method performs the whole detection of the NN. + * + * @param frames frames to run detection on. + * @param cur_batches number of batches to use in inference + * @param save_times if set to true, preprocess, inference and postprocess times + * are saved on a csv file, otherwise not. + * @param times pointer to the output stream where to write times + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation + */ + void update(std::vector& frames, const int cur_batches=1){ + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); + + originalSize.clear(); + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); + { + TKDNN_TSTART + for(int bi=0; biinput_dim; + dim.n = cur_batches; + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + } + + { + TKDNN_TSTART + for(int bi=0; bi> $out_file print_output $? imuodom + test_net shelfnet test_net yolo4 test_net yolo4_berkeley test_net yolo3 diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 71662d3..92a95f2 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -483,6 +483,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Resize *l) { IResizeLayer *lRT = networkRT->addResize(*input); //default is kNEAREST checkNULL(lRT); Dims d{}; + lRT->setResizeMode(ResizeMode(l->mode)); lRT->setOutputDimensions(DimsCHW{l->output_dim.c, l->output_dim.h, l->output_dim.w}); return lRT; } @@ -514,7 +515,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) { ITensor *back_tens = tensors[l->backLayer]; - if(false) //l->backLayer->output_dim.c == l->output_dim.c && !l->mul) FIXME + if(l->backLayer->output_dim.c == l->output_dim.c && !l->mul) { IElementWiseLayer *lRT = networkRT->addElementWise(*input, *back_tens, ElementWiseOperation::kSUM); checkNULL(lRT); diff --git a/src/NetworkViz.cpp b/src/NetworkViz.cpp index 6ac274c..842a26e 100644 --- a/src/NetworkViz.cpp +++ b/src/NetworkViz.cpp @@ -6,23 +6,22 @@ namespace tk { namespace dnn { -cv::Mat vizFloat2colorMap(cv::Mat map) { +cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max) { + + if(min == 0 && max == 0) + cv::minMaxIdx(map, &min, &max); - double min; - double max; - cv::minMaxIdx(map, &min, &max); cv::Mat adjMap; // expand your range to 0..255. Similar to histEq(); map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); //return adjMap; - cv::Mat falseColorsMap; - applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_HOT); + applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_VIRIDIS); return falseColorsMap; } -cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) { +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, double min, double max) { dnnType *data = nullptr; // copy to CPU @@ -38,7 +37,7 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) { cv::Mat grid = cv::Mat(gridSize, CV_8UC3, cv::Scalar(0)); for(int i=0; imode = mode; if(fixed){ output_dim.c = scale_c; output_dim.h = scale_h; diff --git a/tests/shelfnet/shelfnet.cpp b/tests/shelfnet/shelfnet.cpp index ab28d90..7cc1b42 100644 --- a/tests/shelfnet/shelfnet.cpp +++ b/tests/shelfnet/shelfnet.cpp @@ -191,7 +191,7 @@ int main() down_out.push_back(l_last); new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false); - last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU } new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); @@ -231,12 +231,12 @@ int main() new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true); new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); new tk::dnn::Conv2d (&net, 19, 3, 3, 1, 1, 1, 1, conv_out[ci++], false); - // /*up_out[i] =*/ new tk::dnn::Resize(&net, 19, net.input_dim.h, net.input_dim.w, true); + /*up_out[i] =*/ new tk::dnn::Resize(&net, 19, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR); // } - // new tk::dnn::Softmax(&net); + new tk::dnn::Softmax(&net); - const char *output_bin = "shelfnet/debug/conv_out-conv_out.bin"; + const char *output_bin = "shelfnet/debug/softmax.bin"; // Load input dnnType *data; diff --git a/tests/test_rtinference/rtinference.cpp b/tests/test_rtinference/rtinference.cpp index a629168..3eacc37 100644 --- a/tests/test_rtinference/rtinference.cpp +++ b/tests/test_rtinference/rtinference.cpp @@ -31,7 +31,7 @@ int main(int argc, char *argv[]) { std::cout<<"Testing with batchsize: "< batch_frame; std::vector batch_dnn_input; @@ -85,14 +82,8 @@ int main(int argc, char *argv[]) { //inference segNN.update(batch_dnn_input, n_batch); - segNN.draw(); + frame = segNN.draw(); - if(show){ - for(int bi=0; bi< n_batch; ++bi){ - cv::imshow("segmentation", batch_frame[bi]); - cv::waitKey(1); - } - } if(n_batch == 1 && SAVE_RESULT) resultVideo << frame; } diff --git a/include/tkDNN/SegmentationNN.h b/include/tkDNN/SegmentationNN.h index 93be2ff..0289d61 100644 --- a/include/tkDNN/SegmentationNN.h +++ b/include/tkDNN/SegmentationNN.h @@ -17,6 +17,7 @@ #include "tkdnn.h" #include "NetworkViz.h" +#include "kernelsThrust.h" namespace tk { namespace dnn { @@ -33,6 +34,12 @@ class SegmentationNN { dnnType *input_d; float* confidences_h; + float * tmpInputData_d; + float *tmpOutData_d; + float *tmpOutData_h; + + cublasHandle_t cublasHandle; + /** * This method preprocess the image, before feeding it to the NN. * @@ -97,28 +104,14 @@ class SegmentationNN { dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; dataDim_t odim = netRT->output_dim; - - checkCuda(cudaMemcpy(confidences_h, rt_out, odim.tot() * sizeof(float), cudaMemcpyDeviceToHost)); + matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h); + maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w); + checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost)); - for(int i=0;i max_conf){ - max_conf = cur_conf; - max_id = k; - } - } - confidences_h[bi*odim.tot()+0*odim.h*odim.w+i*odim.h+j] = max_id; - } - } dataDim_t vdim = odim; vdim.c = 1; - segmented[bi] = vizData2Mat(confidences_h, vdim, 1024, 0, 18); + segmented[bi] = vizData2Mat(tmpOutData_h, vdim, 1024, 0, 18); }; public: @@ -127,8 +120,12 @@ class SegmentationNN { std::vector classesNames; std::vector segmented; - SegmentationNN() {}; - ~SegmentationNN(){}; + SegmentationNN() { + checkERROR( cublasCreate(&cublasHandle) ); + }; + ~SegmentationNN(){ + checkERROR( cublasDestroy(cublasHandle) ); + }; /** * Method used to inialize the class, allocate memory and compute @@ -151,7 +148,12 @@ class SegmentationNN { checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); - confidences_h = (float *)malloc(netRT->output_dim.tot() * sizeof(float)); + dataDim_t odim = netRT->output_dim; + + checkCuda(cudaMallocHost(&confidences_h, sizeof(float) * odim.tot())); + checkCuda(cudaMalloc(&tmpInputData_d, sizeof(float) * odim.tot())); + checkCuda(cudaMalloc(&tmpOutData_d, sizeof(float) * odim.w*odim.h)); + checkCuda(cudaMallocHost(&tmpOutData_h, sizeof(float) * odim.w*odim.h)); segmented.resize(nBatches); masks.resize(nBatches); @@ -208,7 +210,7 @@ class SegmentationNN { /** * Method to draw boundixg boxes and labels on a frame. */ - void draw(const int cur_batches=1) { + cv::Mat draw(const int cur_batches=1) { for(int i=0; i #include #include #include @@ -9,6 +10,8 @@ #include #include #include +#include + #include "tkdnn.h" @@ -36,4 +39,6 @@ void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_be void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out); +void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w); + #endif //KERNELSTHRUST_H \ No newline at end of file diff --git a/src/kernels/postprocessing.cu b/src/kernels/postprocessing.cu index 3510200..53234a2 100644 --- a/src/kernels/postprocessing.cu +++ b/src/kernels/postprocessing.cu @@ -34,6 +34,25 @@ void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int sortAndTopK_kernel<<>>(src_begin, idsrc, topk_scores, topk_inds, topk_ys, topk_xs, size, K); } +__global__ +void maxElem_kernel(float *src_begin, float *dst_begin, const int n_classes, const int size){ + int i = blockDim.x*blockIdx.x + threadIdx.x; + if (i > size) + return; + + thrust::device_ptr dPbeg ( &src_begin[i*n_classes] ) ; + thrust::device_ptr dPend = dPbeg + n_classes; + thrust::device_ptr result = thrust::max_element(thrust::device,dPbeg, dPend); + + dst_begin[i] = result - dPbeg; +} + +void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w){ + int blocks = (h*w)/32+1; + int threads = 32; + maxElem_kernel<<>>(src_begin, dst_begin, c, h*w); +} + void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys){ thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), clses, thrust::divides()); thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), ids_begin, thrust::modulus()); -- 2.52.0 From 3bf954750273e342fa7d42ab95fe10068f070fab Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 29 Jun 2020 15:11:52 +0200 Subject: [PATCH 061/228] Improve preprocessing Signed-off-by: Micaela Verucchi --- include/tkDNN/SegmentationNN.h | 46 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/include/tkDNN/SegmentationNN.h b/include/tkDNN/SegmentationNN.h index 0289d61..87c7b8c 100644 --- a/include/tkDNN/SegmentationNN.h +++ b/include/tkDNN/SegmentationNN.h @@ -13,8 +13,6 @@ #include #include - - #include "tkdnn.h" #include "NetworkViz.h" #include "kernelsThrust.h" @@ -38,6 +36,8 @@ class SegmentationNN { float *tmpOutData_d; float *tmpOutData_h; + float *mean_d, *stddev_d; + cublasHandle_t cublasHandle; /** @@ -47,23 +47,10 @@ class SegmentationNN { * @param bi batch index */ void preprocess(cv::Mat &frame, const int bi=0) { - frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0); - - cv::split(frame, bgr); - float mean[] = {0.485, 0.456, 0.406}; - float stddev[] = {0.229, 0.224, 0.225}; - for(int i=0; i<3; i++){ - bgr[2-i] -= mean[i]; - bgr[2-i] /= stddev[i]; - } - cv::merge(bgr, 3, frame); - - int crop_size = netRT->input_dim.w; int H = frame.rows; int W = frame.cols; cv::Mat frame_cropped; - cv::Mat mask(frame.size(), CV_8UC3, cv::Scalar(255,255,255)); if(H != W){ @@ -81,17 +68,22 @@ class SegmentationNN { } } - resize(frame_cropped, frame_cropped, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); - resize(mask, mask, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); - masks[bi] = mask.clone(); + tk::dnn::dataDim_t idim = netRT->input_dim; + + resize(frame_cropped, frame_cropped, cv::Size(idim.w, idim.h)); + resize(mask, mask, cv::Size(idim.w, idim.h)); + masks[bi] = mask; cv::split(frame_cropped, bgr); - for (int i = 0; i < netRT->input_dim.c; i++){ + for (int i = 0; i < idim.c; i++){ int idx = i * frame_cropped.rows * frame_cropped.cols; - int ch = netRT->input_dim.c-1 -i; - memcpy((void *)&input[idx + netRT->input_dim.tot()*bi], (void *)bgr[ch].data, frame_cropped.rows * frame_cropped.cols * sizeof(dnnType)); + int ch = idim.c-1 -i; + memcpy((void *)&input[idx + idim.tot()*bi], (void *)bgr[ch].data, frame_cropped.rows * frame_cropped.cols * sizeof(dnnType)); } - checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); + + checkCuda(cudaMemcpyAsync(input_d+ idim.tot()*bi, input + idim.tot()*bi, idim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); + + normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d); } /** @@ -157,6 +149,16 @@ class SegmentationNN { segmented.resize(nBatches); masks.resize(nBatches); + + std::vector mean = {0.485, 0.456, 0.406}; + std::vector stddev = {0.229, 0.224, 0.225}; + + checkCuda(cudaMalloc(&mean_d, sizeof(float) * mean.size())); + checkCuda(cudaMalloc(&stddev_d, sizeof(float) * stddev.size())); + + checkCuda(cudaMemcpyAsync(mean_d, mean.data(), mean.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream)); + checkCuda(cudaMemcpyAsync(stddev_d, stddev.data(), stddev.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream)); + } /** -- 2.52.0 From 61aa24c6b7716b833c8a0840dc0c47a00ca7e8ff Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 30 Jun 2020 15:19:03 +0200 Subject: [PATCH 062/228] yolov4tiny works on CUDNN Signed-off-by: Micaela Verucchi --- include/tkDNN/DarknetParser.h | 1 + include/tkDNN/Layer.h | 4 +- src/DarknetParser.cpp | 4 +- src/Route.cpp | 10 +- tests/darknet/cfg/yolo4tiny.cfg | 281 ++++++++++++++++++++++++++++++++ tests/darknet/yolo4tiny.cpp | 33 ++++ 6 files changed, 328 insertions(+), 5 deletions(-) create mode 100644 tests/darknet/cfg/yolo4tiny.cfg create mode 100644 tests/darknet/yolo4tiny.cpp diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index f36469b..29d1e8e 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -11,6 +11,7 @@ namespace tk { namespace dnn { int channels = 3; int batch_normalize=0; int groups = 1; + int group_id = 0; int filters=1; int size_x=1; int size_y=1; diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 9bd8432..bd544b2 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -509,7 +509,7 @@ public: class Route : public Layer { public: - Route(Network *net, Layer **layers, int layers_n); + Route(Network *net, Layer **layers, int layers_n, int groups = 1, int group_id = 0); virtual ~Route(); virtual layerType_t getLayerType() { return LAYER_ROUTE; }; @@ -519,6 +519,8 @@ public: static const int MAX_LAYERS = 32; Layer *layers[MAX_LAYERS]; //ids of layers to be merged int layers_n; //number of layers + int groups; + int group_id; }; diff --git a/src/DarknetParser.cpp b/src/DarknetParser.cpp index 5092595..7bbc7ba 100644 --- a/src/DarknetParser.cpp +++ b/src/DarknetParser.cpp @@ -75,6 +75,8 @@ namespace tk { namespace dnn { fields.coords = std::stoi(value); else if(name.find("groups") != std::string::npos) fields.groups = std::stoi(value); + else if(name.find("group_id") != std::string::npos) + fields.group_id = std::stoi(value); else if(name.find("scale_x_y") != std::string::npos) fields.scale_xy = std::stof(value); else if(name.find("from") != std::string::npos) @@ -148,7 +150,7 @@ namespace tk { namespace dnn { //std::cout<<"Route to "<getLayerName()<<"\n"; layers.push_back(netLayers[layerIdx]); } - netLayers.push_back(new tk::dnn::Route(net, layers.data(), layers.size())); + netLayers.push_back(new tk::dnn::Route(net, layers.data(), layers.size(), f.groups, f.group_id)); } else if(f.type == "reorg") { netLayers.push_back(new tk::dnn::Reorg(net, f.stride_x)); diff --git a/src/Route.cpp b/src/Route.cpp index 39bb14e..816566e 100644 --- a/src/Route.cpp +++ b/src/Route.cpp @@ -5,7 +5,7 @@ namespace tk { namespace dnn { -Route::Route(Network *net, Layer **layers, int layers_n) : Layer(net) { +Route::Route(Network *net, Layer **layers, int layers_n, int groups, int group_id) : Layer(net) { // copy input layers if(layers_n > MAX_LAYERS) { @@ -15,6 +15,8 @@ Route::Route(Network *net, Layer **layers, int layers_n) : Layer(net) { this->layers[i] = layers[i]; } this->layers_n = layers_n; + this->groups = groups; + this->group_id = group_id; //get dims output_dim.l = 1; @@ -32,6 +34,7 @@ Route::Route(Network *net, Layer **layers, int layers_n) : Layer(net) { output_dim.c += layers[i]->output_dim.c; } + output_dim.c /= this->groups; input_dim = output_dim; checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); @@ -49,8 +52,9 @@ dnnType* Route::infer(dataDim_t &dim, dnnType* srcData) { for(int i=0; idstData; int in_dim = layers[i]->output_dim.tot(); - checkCuda( cudaMemcpy(dstData + offset, input, in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); - offset += in_dim; + int part_in_dim = in_dim / this->groups; + checkCuda( cudaMemcpy(dstData + offset, input + this->group_id*part_in_dim, part_in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + offset += part_in_dim; } //update data dimensions diff --git a/tests/darknet/cfg/yolo4tiny.cfg b/tests/darknet/cfg/yolo4tiny.cfg new file mode 100644 index 0000000..dc6f5bf --- /dev/null +++ b/tests/darknet/cfg/yolo4tiny.cfg @@ -0,0 +1,281 @@ +[net] +# Testing +#batch=1 +#subdivisions=1 +# Training +batch=64 +subdivisions=1 +width=416 +height=416 +channels=3 +momentum=0.9 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.00261 +burn_in=1000 +max_batches = 500200 +policy=steps +steps=400000,450000 +scales=.1,.1 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=leaky + +[route] +layers=-1 +groups=2 +group_id=1 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1,-2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -6,-1 + +[maxpool] +size=2 +stride=2 + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[route] +layers=-1 +groups=2 +group_id=1 + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1,-2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -6,-1 + +[maxpool] +size=2 +stride=2 + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[route] +layers=-1 +groups=2 +group_id=1 + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1,-2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -6,-1 + +[maxpool] +size=2 +stride=2 + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +################################## + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + + +[yolo] +mask = 3,4,5 +anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319 +classes=80 +num=6 +jitter=.3 +scale_x_y = 1.05 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +ignore_thresh = .7 +truth_thresh = 1 +random=0 +resize=1.5 +nms_kind=greedynms +beta_nms=0.6 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 23 + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + +[yolo] +mask = 1,2,3 +anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319 +classes=80 +num=6 +jitter=.3 +scale_x_y = 1.05 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +ignore_thresh = .7 +truth_thresh = 1 +random=0 +resize=1.5 +nms_kind=greedynms +beta_nms=0.6 diff --git a/tests/darknet/yolo4tiny.cpp b/tests/darknet/yolo4tiny.cpp new file mode 100644 index 0000000..d9011a8 --- /dev/null +++ b/tests/darknet/yolo4tiny.cpp @@ -0,0 +1,33 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4tiny"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer30_out.bin", + bin_path + "/debug/layer37_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4tiny.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + // tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, nullptr); + net->releaseLayers(); + delete net; + // delete netRT; + return ret; +} -- 2.52.0 From fe2e4eae92324d4c03482e20b5746ebc6aed75c5 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 30 Jun 2020 15:52:58 +0200 Subject: [PATCH 063/228] yolo4tiny works on tensorRT :dolphin: :dolphin: :dolphin: :dolphin: Signed-off-by: Micaela Verucchi --- include/tkDNN/NetworkRT.h | 2 +- include/tkDNN/pluginsRT/RouteRT.h | 17 ++++++++++++----- src/NetworkRT.cpp | 19 +++++++++++-------- tests/darknet/yolo4tiny.cpp | 6 +++--- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index ee1f728..66b4f3d 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -28,7 +28,7 @@ using namespace nvinfer1; #include "pluginsRT/ActivationMishRT.h" #include "pluginsRT/ReorgRT.h" #include "pluginsRT/RegionRT.h" -//#include "pluginsRT/RouteRT.h" +#include "pluginsRT/RouteRT.h" #include "pluginsRT/ShortcutRT.h" #include "pluginsRT/YoloRT.h" #include "pluginsRT/UpsampleRT.h" diff --git a/include/tkDNN/pluginsRT/RouteRT.h b/include/tkDNN/pluginsRT/RouteRT.h index 0e94a97..263893f 100644 --- a/include/tkDNN/pluginsRT/RouteRT.h +++ b/include/tkDNN/pluginsRT/RouteRT.h @@ -8,7 +8,9 @@ class RouteRT : public IPlugin { */ public: - RouteRT() { + RouteRT(int groups, int group_id) { + this->groups = groups; + this->group_id = group_id; } ~RouteRT(){ @@ -22,7 +24,7 @@ public: Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override { int out_c = 0; for(int i=0; i(inputs[i]); int in_dim = c_in[i]*h*w; - checkCuda( cudaMemcpyAsync(dstData + offset, input, in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream) ); - offset += in_dim; + int part_in_dim = in_dim / this->groups; + checkCuda( cudaMemcpyAsync(dstData + offset, input + this->group_id*part_in_dim, part_in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream) ); + offset += part_in_dim; } return 0; @@ -65,11 +69,13 @@ public: virtual size_t getSerializationSize() override { - return (4+MAX_INPUTS)*sizeof(int); + return (6+MAX_INPUTS)*sizeof(int); } virtual void serialize(void* buffer) override { char *buf = reinterpret_cast(buffer); + tk::dnn::writeBUF(buf, groups); + tk::dnn::writeBUF(buf, group_id); tk::dnn::writeBUF(buf, in); for(int i=0; iaddConcatenation(tens, l->layers_n); - //IPlugin *plugin = new RouteRT(); - //IPluginLayer *lRT = networkRT->addPlugin(tens, l->layers_n, *plugin); - checkNULL(lRT); + if(l->groups > 1){ + IPlugin *plugin = new RouteRT(l->groups, l->group_id); + IPluginLayer *lRT = networkRT->addPlugin(tens, l->layers_n, *plugin); + checkNULL(lRT); + return lRT; + } + IConcatenationLayer *lRT = networkRT->addConcatenation(tens, l->layers_n); + checkNULL(lRT); return lRT; } @@ -766,9 +769,9 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa r->w = readBUF(buf); return r; } -/* + if(name.find("Route") == 0) { - RouteRT *r = new RouteRT(); + RouteRT *r = new RouteRT(readBUF(buf),readBUF(buf)); r->in = readBUF(buf); for(int i=0; ic_in[i] = readBUF(buf); @@ -777,7 +780,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa r->w = readBUF(buf); return r; } -*/ + if(name.find("Deformable") == 0) { DeformableConvRT *r = new DeformableConvRT(readBUF(buf), readBUF(buf), readBUF(buf), readBUF(buf), readBUF(buf), readBUF(buf), diff --git a/tests/darknet/yolo4tiny.cpp b/tests/darknet/yolo4tiny.cpp index d9011a8..44fbac8 100644 --- a/tests/darknet/yolo4tiny.cpp +++ b/tests/darknet/yolo4tiny.cpp @@ -23,11 +23,11 @@ int main() { net->print(); //convert network to tensorRT - // tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); - int ret = testInference(input_bins, output_bins, net, nullptr); + int ret = testInference(input_bins, output_bins, net, netRT); net->releaseLayers(); delete net; - // delete netRT; + delete netRT; return ret; } -- 2.52.0 From 04602f395236d993d79cff7d1be5ced3ae738b0b Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Tue, 30 Jun 2020 19:37:16 +0200 Subject: [PATCH 064/228] Yolo4-tiny batched fix #59 --- README.md | 2 ++ include/tkDNN/pluginsRT/RouteRT.h | 18 ++++++++++-------- scripts/test_all_tests.sh | 1 + 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 19b98a2..3ff6fe7 100644 --- a/README.md +++ b/README.md @@ -317,6 +317,8 @@ This demo also creates a json file named ```net_name_COCO_res.json``` containing | resnet101_cnet | Centernet (Resnet101 backend)4 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download) | | csresnext50-panet-spp | Cross Stage Partial Network 7 | [COCO 2014](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download) | | yolo4 | Yolov4 8 | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) | +| yolo4_berkeley | Yolov4 8 | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 540x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) | +| yolo4tiny | Yolov4 tiny | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | ## References diff --git a/include/tkDNN/pluginsRT/RouteRT.h b/include/tkDNN/pluginsRT/RouteRT.h index 263893f..23f30b7 100644 --- a/include/tkDNN/pluginsRT/RouteRT.h +++ b/include/tkDNN/pluginsRT/RouteRT.h @@ -52,16 +52,18 @@ public: } virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override { - + dnnType *dstData = reinterpret_cast(outputs[0]); - int offset = 0; - for(int i=0; i(inputs[i]); - int in_dim = c_in[i]*h*w; - int part_in_dim = in_dim / this->groups; - checkCuda( cudaMemcpyAsync(dstData + offset, input + this->group_id*part_in_dim, part_in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream) ); - offset += part_in_dim; + for(int b=0; b(inputs[i]); + int in_dim = c_in[i]*h*w; + int part_in_dim = in_dim / this->groups; + checkCuda( cudaMemcpyAsync(dstData + b*c*w*h + offset, input + b*c*w*h*groups + this->group_id*part_in_dim, part_in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream) ); + offset += part_in_dim; + } } return 0; diff --git a/scripts/test_all_tests.sh b/scripts/test_all_tests.sh index 193a90c..770aa22 100644 --- a/scripts/test_all_tests.sh +++ b/scripts/test_all_tests.sh @@ -74,6 +74,7 @@ do test_net yolo4 test_net yolo4_berkeley + test_net yolo4tiny test_net yolo3 test_net yolo3_berkeley test_net yolo3_coco4 -- 2.52.0 From a5cc4e3edada046902cb9fd3eea0ad1efe59970a Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Wed, 1 Jul 2020 10:53:24 +0200 Subject: [PATCH 065/228] Add berkeley test, add weights for shelfnets Signed-off-by: Micaela Verucchi --- CMakeLists.txt | 3 + demo/demo/seg_demo.cpp | 9 +- tests/shelfnet/shelfnet.cpp | 2 +- tests/shelfnet/shelfnet_berkeley.cpp | 295 +++++++++++++++++++++++++++ 4 files changed, 305 insertions(+), 4 deletions(-) create mode 100644 tests/shelfnet/shelfnet_berkeley.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 88e94e3..9e9c27c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,9 @@ target_link_libraries(test_dla34_cnet tkDNN) add_executable(test_shelfnet tests/shelfnet/shelfnet.cpp) target_link_libraries(test_shelfnet tkDNN) +add_executable(test_shelfnet_berkeley tests/shelfnet/shelfnet_berkeley.cpp) +target_link_libraries(test_shelfnet_berkeley tkDNN) + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) diff --git a/demo/demo/seg_demo.cpp b/demo/demo/seg_demo.cpp index c1ee43b..0737ad7 100644 --- a/demo/demo/seg_demo.cpp +++ b/demo/demo/seg_demo.cpp @@ -29,9 +29,12 @@ int main(int argc, char *argv[]) { int n_batch = 1; if(argc > 3) n_batch = atoi(argv[3]); - bool show = false; + int n_classes = 19; if(argc > 4) - show = atoi(argv[4]); + n_classes = atoi(argv[4]); + bool show = false; + if(argc > 5) + show = atoi(argv[5]); if(n_batch < 1 || n_batch > 64) FatalError("Batch dim not supported"); @@ -39,7 +42,7 @@ int main(int argc, char *argv[]) { if(!show) SAVE_RESULT = true; - int n_classes = 19; + tk::dnn::SegmentationNN segNN; segNN.init(net, n_classes, n_batch); diff --git a/tests/shelfnet/shelfnet.cpp b/tests/shelfnet/shelfnet.cpp index 7cc1b42..48cad04 100644 --- a/tests/shelfnet/shelfnet.cpp +++ b/tests/shelfnet/shelfnet.cpp @@ -83,7 +83,7 @@ const char *trans[] = { int main() { - // downloadWeightsifDoNotExist(input_bin, "shelfnet", "https://cloud.hipert.unimore.it/s/x4ZfxBKN23zAJQp/download"); + downloadWeightsifDoNotExist(input_bin, "shelfnet", "https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download"); int classes = 19; diff --git a/tests/shelfnet/shelfnet_berkeley.cpp b/tests/shelfnet/shelfnet_berkeley.cpp new file mode 100644 index 0000000..fe81191 --- /dev/null +++ b/tests/shelfnet/shelfnet_berkeley.cpp @@ -0,0 +1,295 @@ +#include +#include +#include + +#include "tkdnn.h" +#include "NetworkViz.h" + + +const char *input_bin = "shelfnet_berkeley/debug/input.bin"; + +const char *backbone[] = { + "shelfnet_berkeley/layers/backbone-conv1.bin", + "shelfnet_berkeley/layers/backbone-layer1-0-conv1.bin", + "shelfnet_berkeley/layers/backbone-layer1-0-conv2.bin", + "shelfnet_berkeley/layers/backbone-layer1-1-conv1.bin", + "shelfnet_berkeley/layers/backbone-layer1-1-conv2.bin", + "shelfnet_berkeley/layers/backbone-layer2-0-conv1.bin", + "shelfnet_berkeley/layers/backbone-layer2-0-conv2.bin", + "shelfnet_berkeley/layers/backbone-layer2-0-downsample-0.bin", + "shelfnet_berkeley/layers/backbone-layer2-1-conv1.bin", + "shelfnet_berkeley/layers/backbone-layer2-1-conv2.bin", + "shelfnet_berkeley/layers/backbone-layer3-0-conv1.bin", + "shelfnet_berkeley/layers/backbone-layer3-0-conv2.bin", + "shelfnet_berkeley/layers/backbone-layer3-0-downsample-0.bin", + "shelfnet_berkeley/layers/backbone-layer3-1-conv1.bin", + "shelfnet_berkeley/layers/backbone-layer3-1-conv2.bin", + "shelfnet_berkeley/layers/backbone-layer4-0-conv1.bin", + "shelfnet_berkeley/layers/backbone-layer4-0-conv2.bin", + "shelfnet_berkeley/layers/backbone-layer4-0-downsample-0.bin", + "shelfnet_berkeley/layers/backbone-layer4-1-conv1.bin", + "shelfnet_berkeley/layers/backbone-layer4-1-conv2.bin"}; + +const char *conv_out[] = { + "shelfnet_berkeley/layers/conv_out-conv-conv.bin", + "shelfnet_berkeley/layers/conv_out-conv_out.bin", + "shelfnet_berkeley/layers/conv_out16-conv-conv.bin", + "shelfnet_berkeley/layers/conv_out16-conv_out.bin", + "shelfnet_berkeley/layers/conv_out32-conv-conv.bin", + "shelfnet_berkeley/layers/conv_out32-conv_out.bin" + }; + +const char *decoder[] = { + "shelfnet_berkeley/layers/decoder-bottom-conv1.bin", + "shelfnet_berkeley/layers/decoder-bottom-conv12.bin", + "shelfnet_berkeley/layers/decoder-up_conv_list-0-conv-conv.bin", + "shelfnet_berkeley/layers/decoder-up_conv_list-0-conv_atten.bin", + "shelfnet_berkeley/layers/decoder-up_dense_list-0-conv.bin", + "shelfnet_berkeley/layers/decoder-up_conv_list-1-conv-conv.bin", + "shelfnet_berkeley/layers/decoder-up_conv_list-1-conv_atten.bin", + "shelfnet_berkeley/layers/decoder-up_dense_list-1-conv.bin" + }; + + +const char *ladder[] = { + "shelfnet_berkeley/layers/ladder-inconv-conv1.bin", + "shelfnet_berkeley/layers/ladder-inconv-conv12.bin", + "shelfnet_berkeley/layers/ladder-down_module_list-0-conv1.bin", + "shelfnet_berkeley/layers/ladder-down_module_list-0-conv12.bin", + "shelfnet_berkeley/layers/ladder-down_conv_list-0.bin", + + "shelfnet_berkeley/layers/ladder-down_module_list-1-conv1.bin", + "shelfnet_berkeley/layers/ladder-down_module_list-1-conv12.bin", + "shelfnet_berkeley/layers/ladder-down_conv_list-1.bin", + + "shelfnet_berkeley/layers/ladder-bottom-conv1.bin", + "shelfnet_berkeley/layers/ladder-bottom-conv12.bin", + + + + "shelfnet_berkeley/layers/ladder-up_conv_list-0-conv-conv.bin", + "shelfnet_berkeley/layers/ladder-up_conv_list-0-conv_atten.bin", + "shelfnet_berkeley/layers/ladder-up_dense_list-0-conv.bin", + + + "shelfnet_berkeley/layers/ladder-up_conv_list-1-conv-conv.bin", + "shelfnet_berkeley/layers/ladder-up_conv_list-1-conv_atten.bin", + "shelfnet_berkeley/layers/ladder-up_dense_list-1-conv.bin"}; + +const char *trans[] = { + "shelfnet_berkeley/layers/trans1-conv.bin", + "shelfnet_berkeley/layers/trans2-conv.bin", + "shelfnet_berkeley/layers/trans3-conv.bin"}; +int main() +{ + + downloadWeightsifDoNotExist(input_bin, "shelfnet_berkeley", "https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download"); + + int classes = 20; + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1); + tk::dnn::Network net(dim); + + int bi = 0, di = 0, li = 0, ci = 0; + new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); + + + + for(int i=0; i<2; ++i){ + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + } + + std::vector features; + for(int i=0;i<3;++i){ + int out_channel = pow(2,7+i); + std::cout< up_out; + //bottom + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + std::cout<output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, features[1-i]); + + //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + //LADDER + + std::vector down_out; + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + + for(int i=0; i<2;++i){ + int out_channel = pow(2,6+i); + tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]); + + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, l_last); + l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + down_out.push_back(l_last); + + new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU + } + + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.clear(); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + + new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, down_out[1-i]); + + // //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + + // for(int i=2;i>=0;--i){ + // new tk::dnn::Route(&net, &up_out[i], 1); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, classes, 3, 3, 1, 1, 1, 1, conv_out[ci++], false); + /*up_out[i] =*/ new tk::dnn::Resize(&net, classes, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR); + // } + + new tk::dnn::Softmax(&net); + + const char *output_bin = "shelfnet_berkeley/debug/softmax.bin"; + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + std::cout<<"Input:"< Date: Mon, 13 Jul 2020 19:51:09 +0200 Subject: [PATCH 072/228] Update README.md --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index a3edf3c..aec202a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,7 +1,7 @@ # Use the prebuilt image ``` # build image -docker build -t tkdnn:build -f Dockerfile-f Dockerfile . +docker build -t tkdnn:build -f Dockerfile . ``` # Build Base Docker image -- 2.52.0 From c4aad7fe95e0f8ed45fd175d565d1926d2880f6c Mon Sep 17 00:00:00 2001 From: tk Date: Thu, 16 Jul 2020 18:16:09 +0200 Subject: [PATCH 073/228] Patch for CUDNN 8.0.1 Signed-off-by: tk --- include/tkDNN/Layer.h | 4 ++-- src/Conv2d.cpp | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index bd544b2..f2ec56d 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -273,8 +273,8 @@ public: protected: cudnnFilterDescriptor_t filterDesc; cudnnConvolutionDescriptor_t convDesc; - cudnnConvolutionFwdAlgo_t algo; - cudnnConvolutionBwdDataAlgo_t bwAlgo; + cudnnConvolutionFwdAlgoPerf_t algo; + cudnnConvolutionBwdDataAlgoPerf_t bwAlgo; cudnnTensorDescriptor_t biasTensorDesc; void initCUDNN(bool back = false); diff --git a/src/Conv2d.cpp b/src/Conv2d.cpp index 4704c66..595fec7 100644 --- a/src/Conv2d.cpp +++ b/src/Conv2d.cpp @@ -63,23 +63,23 @@ void Conv2d::initCUDNN(bool back) { workSpace = NULL; ws_sizeInBytes = 0; if(back) { - checkCUDNN( cudnnGetConvolutionBackwardDataAlgorithm(net->cudnnHandle, - filterDesc, dstTensor, convDesc, srcTensor, - CUDNN_CONVOLUTION_BWD_DATA_PREFER_FASTEST, 0, &bwAlgo) ); + checkCUDNN( cudnnGetConvolutionBackwardDataAlgorithm_v7(net->cudnnHandle, + filterDesc, dstTensor, convDesc, srcTensor, 1, 0, &bwAlgo) ); checkCUDNN(cudnnGetConvolutionBackwardDataWorkspaceSize(net->cudnnHandle, - filterDesc, dstTensor, convDesc, srcTensor, - bwAlgo, &ws_sizeInBytes)); + filterDesc, dstTensor, convDesc, srcTensor, + bwAlgo.algo, &ws_sizeInBytes)); + // invert tensors srcTensorDesc = dstTensor; dstTensorDesc = srcTensor; } else { - checkCUDNN( cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle, - srcTensor, filterDesc, convDesc, dstTensor, - CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo) ); - checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle, - srcTensor, filterDesc, convDesc, dstTensor, - algo, &ws_sizeInBytes)); + checkCUDNN( cudnnGetConvolutionForwardAlgorithm_v7(net->cudnnHandle, + srcTensor, filterDesc, convDesc, dstTensor, + 1, 0, &algo) ); + checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle, + srcTensor, filterDesc, convDesc, dstTensor, + algo.algo, &ws_sizeInBytes)); } } @@ -91,12 +91,12 @@ void Conv2d::inferCUDNN(dnnType* srcData, bool back) { checkCUDNN(cudnnConvolutionBackwardData(net->cudnnHandle, &alpha, filterDesc, data_d, srcTensorDesc, srcData, - convDesc, bwAlgo, workSpace, ws_sizeInBytes, + convDesc, bwAlgo.algo, workSpace, ws_sizeInBytes, &beta, dstTensorDesc, dstData)); } else { checkCUDNN(cudnnConvolutionForward(net->cudnnHandle, &alpha, srcTensorDesc, srcData, filterDesc, - data_d, convDesc, algo, workSpace, ws_sizeInBytes, + data_d, convDesc, algo.algo, workSpace, ws_sizeInBytes, &beta, dstTensorDesc, dstData)); } -- 2.52.0 From 6a68f19b2ceb61542fe87533fb193f696d30c664 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 16 Jul 2020 18:37:37 +0200 Subject: [PATCH 074/228] Fix patch from @ahmedius2 , tkDNN now supports CUDNN 8.0.1 (Fix #74) Signed-off-by: Micaela Verucchi Francesco Gatti --- src/Conv2d.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Conv2d.cpp b/src/Conv2d.cpp index 595fec7..b57cf58 100644 --- a/src/Conv2d.cpp +++ b/src/Conv2d.cpp @@ -62,9 +62,10 @@ void Conv2d::initCUDNN(bool back) { // init workspace workSpace = NULL; ws_sizeInBytes = 0; + int algo_count = 0; if(back) { checkCUDNN( cudnnGetConvolutionBackwardDataAlgorithm_v7(net->cudnnHandle, - filterDesc, dstTensor, convDesc, srcTensor, 1, 0, &bwAlgo) ); + filterDesc, dstTensor, convDesc, srcTensor, 1, &algo_count, &bwAlgo) ); checkCUDNN(cudnnGetConvolutionBackwardDataWorkspaceSize(net->cudnnHandle, filterDesc, dstTensor, convDesc, srcTensor, bwAlgo.algo, &ws_sizeInBytes)); @@ -74,13 +75,17 @@ void Conv2d::initCUDNN(bool back) { srcTensorDesc = dstTensor; dstTensorDesc = srcTensor; } else { + checkCUDNN( cudnnGetConvolutionForwardAlgorithm_v7(net->cudnnHandle, srcTensor, filterDesc, convDesc, dstTensor, - 1, 0, &algo) ); + 1, &algo_count, &algo) ); checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle, srcTensor, filterDesc, convDesc, dstTensor, algo.algo, &ws_sizeInBytes)); } + + if(algo_count < 1) + FatalError("Cannot retrieve convolutional algo"); } void Conv2d::inferCUDNN(dnnType* srcData, bool back) { -- 2.52.0 From f4970d1e6faab505c2caf1d6833cf7490a971a0e Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Fri, 17 Jul 2020 14:37:10 +0200 Subject: [PATCH 075/228] Update README.md --- README.md | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3ff6fe7..a1b5b16 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # tkDNN -tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier and several discrete GPU. +tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier, Nano and several discrete GPUs. The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training. -If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it or refer to https://hipert.unimore.it/ . +If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it and micaela.verucchi@unimore.it or refer to https://hipert.unimore.it/ . ``` Accepted paper @ IRC 2020, will soon be published. @@ -175,15 +175,25 @@ All models from darknet are now parsed directly from cfg, you still need to expo mish -## Run the demo +## Run the demo +This is an example using yolov4. -To run the an object detection demo follow these steps (example with yolov3): +To run the an object detection first create the .rt file by running: ``` -rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files -./test_yolo3 # run the yolo test (is slow) -./demo yolo3_fp32.rt ../demo/yolo_test.mp4 y +rm yolo4_fp32.rt # be sure to delete(or move) old tensorRT files +./test_yolo4 # run the yolo test (is slow) ``` -In general the demo program takes 4 parameters: +If you get problems in the creation, try to check the error activating the debug of TensorRT in this way: +``` +cmake .. -DDEBUG=True +make +``` + +Once you have succesfully created your rt file, run the demo: +``` +./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y +``` +In general the demo program takes 6 parameters: ``` ./demo ``` @@ -197,6 +207,7 @@ where N.b. By default it is used FP32 inference + ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) ### FP16 inference -- 2.52.0 From 286e7773000ecfa461c8adec9ee1d07b68e80a99 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Wed, 22 Jul 2020 19:21:46 +0200 Subject: [PATCH 076/228] Improved segmentation results, removed resize, code to reorder Signed-off-by: Micaela Verucchi --- demo/demo/seg_demo.cpp | 93 ++++++++----------- include/tkDNN/SegmentationNN.h | 157 ++++++++++++++++++++++++++++++++- 2 files changed, 191 insertions(+), 59 deletions(-) diff --git a/demo/demo/seg_demo.cpp b/demo/demo/seg_demo.cpp index 658e0b9..35b4fa8 100644 --- a/demo/demo/seg_demo.cpp +++ b/demo/demo/seg_demo.cpp @@ -14,20 +14,18 @@ void sig_handler(int signo) { gRun = false; } -void writePred(const std::string& images_names, const std::string& gt_folder, const std::string& out_folder, tk::dnn::SegmentationNN& segNN){ +void writePred(const std::string& images_names, const std::string& gt_folder, const std::string& out_folder, tk::dnn::SegmentationNN& segNN, int& width, int& height, bool show=false){ std::ifstream all_gt(images_names); std::string filename; cv::Mat frame; - std::vector batch_frame; - std::vector batch_dnn_input; for (; std::getline(all_gt, filename); ) { std::cout< 4) n_classes = atoi(argv[4]); - bool show = false; + bool show = true; if(argc > 5) show = atoi(argv[5]); bool write_pred = false; @@ -64,80 +62,65 @@ int main(int argc, char *argv[]) { tk::dnn::SegmentationNN segNN; segNN.init(net, n_classes, n_batch); + int height = 0, width = 0; + if(write_pred){ std::string gt_folder = "../demo/CityScapes_val/images/"; std::string images_names = "../demo/CityScapes_val/all_images.txt"; std::string out_folder = "seg/"; - writePred(images_names, gt_folder, out_folder, segNN); - return 0; + writePred(images_names, gt_folder, out_folder, segNN, width, height, show); } + else{ + if(!show) + SAVE_RESULT = true; - if(!show) - SAVE_RESULT = true; + gRun = true; - gRun = true; + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; - cv::VideoCapture cap(input); - if(!cap.isOpened()) - gRun = false; - else - std::cout<<"camera started\n"; + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(1024, 1024)); + } - cv::VideoWriter resultVideo; - if(SAVE_RESULT) { - int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); - int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); - resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(1024, 1024)); - } - - cv::Mat frame; - std::vector batch_frame; - std::vector batch_dnn_input; - int height = 0, width = 0; - - while(gRun) { - batch_dnn_input.clear(); - batch_frame.clear(); - - for(int bi=0; bi< n_batch; ++bi){ + cv::Mat frame; + while(gRun) { cap >> frame; if(!frame.data) break; height = frame.rows; width = frame.cols; - batch_frame.push_back(frame); - // this will be resized to the net format - batch_dnn_input.push_back(frame.clone()); - } - if(!frame.data) - break; - - //inference - segNN.update(batch_dnn_input, n_batch); - frame = segNN.draw(); + //inference + segNN.updateOriginal(frame); + if(show) + segNN.draw(); - if(n_batch == 1 && SAVE_RESULT) - resultVideo << frame; + if(SAVE_RESULT) + resultVideo << segNN.segmented[0]; + } } std::cout<<"segmentation end\n"; double mean = 0, mean_pre = 0, mean_post = 0; std::cout< splitted_frames; + int H, W, net_H, net_W; + int top = 0, bottom = 0, left = 0, right = 0; + std::vector> pos; + + { + TKDNN_TSTART + cv::Size original_size = frame.size(); + + frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0); + H = frame.rows; + W = frame.cols; + net_H = netRT->input_dim.h; + net_W = netRT->input_dim.w; + + cv::Mat frame_cropped; + + if( H <= net_H && W <= net_W ){ // smaller size wrt network + top = (net_H - H)/2; + bottom = net_H - H - top ; + left = (net_W - W)/2; + right = net_W - W - left ; + cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) ); + splitted_frames.push_back(frame_cropped); + } + else{ //bigger size wrt network + + + if(H < net_H || W < net_W){ + if(H < net_H){ + top = (net_H - H)/2; + bottom = net_H - H - top ; + } + else{ + left = (net_W - W)/2; + right = net_W - W - left ; + } + cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0)); + } + + for(int x=0; x+net_W<=W ;){ + for(int y=0; y+net_H <=H ; ){ + cv::Rect roi(x, y, net_W, net_H); + cv::Mat image_roi = frame(roi); + splitted_frames.push_back(image_roi); + pos.push_back(std::make_pair(x,y)); + + y += net_H; + if(y == H) + break; + if(y + net_H > H) y = H - net_H; + } + x += net_W; + if(x == W) + break; + if(x + net_W > W) x = W - net_W; + } + } + + tk::dnn::dataDim_t idim = netRT->input_dim; + + if(splitted_frames.size()> nBatches) + FatalError(std::to_string(splitted_frames.size()) + " min batches required"); + + for(int bi=0; bistream)); + normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d); + } + TKDNN_TSTOP + stats_pre.push_back(t_ns); + } + + tk::dnn::dataDim_t dim = netRT->input_dim; + dim.n = splitted_frames.size(); + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + } + + dataDim_t odim = netRT->output_dim; + + std::vector out_img; + + { + TKDNN_TSTART + + for(int bi=0; bibuffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + + matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h); + maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w); + checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost)); + + dataDim_t vdim = odim; + vdim.c = 1; + + cv::Mat colored; + + if(apply_colormap) + colored = vizData2Mat(tmpOutData_h, vdim, 1024, 0, 18); + else{ + cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h); + colored_fp32.convertTo(colored, CV_8UC1); + } + out_img.push_back(colored); + } + + + cv::Mat seg(frame.size(), out_img[0].type()); + if(out_img.size() == 1) + { + cv::Rect roi(left, top, W, H); + seg = out_img[0](roi); + } + else{ + int bi=0; + + if(top == 0 && left == 0){ + + for(int i=0; i Date: Mon, 27 Jul 2020 13:45:42 +0200 Subject: [PATCH 077/228] Resolve detection objects pick by prob threshold. Before this it will only pick the last object with prob > thresh wich is absolutely wrong Now it picks all the objects with prob > thesh. fixes #94 --- src/Yolo3Detection.cpp | 47 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index c76af20..27f393f 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -113,34 +113,35 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ int x1 = (b.x+b.w/2.); int y0 = (b.y-b.h/2.); int y1 = (b.y+b.h/2.); - int obj_class = -1; - float prob = 0; + for(int c=0; c= confThreshold) { - obj_class = c; - prob = dets[j].prob[c]; + int obj_class = c; + float prob = dets[j].prob[c]; + + // convert to image coords + x0 = x_ratio*x0; + x1 = x_ratio*x1; + y0 = y_ratio*y0; + y1 = y_ratio*y1; + + tk::dnn::box res; + res.cl = obj_class; + res.prob = prob; + res.x = x0; + res.y = y0; + res.w = x1 - x0; + res.h = y1 - y0; + + // FIXME: this shuld be useless + // if(mAP) + // for(int c=0; c= 0) { - // convert to image coords - x0 = x_ratio*x0; - x1 = x_ratio*x1; - y0 = y_ratio*y0; - y1 = y_ratio*y1; - - tk::dnn::box res; - res.cl = obj_class; - res.prob = prob; - res.x = x0; - res.y = y0; - res.w = x1 - x0; - res.h = y1 - y0; - if(mAP) - for(int c=0; c Date: Mon, 27 Jul 2020 13:52:39 +0200 Subject: [PATCH 078/228] fix coords convert --- src/Yolo3Detection.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index 27f393f..606c6d1 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -114,17 +114,17 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ int y0 = (b.y-b.h/2.); int y1 = (b.y+b.h/2.); + // convert to image coords + x0 = x_ratio*x0; + x1 = x_ratio*x1; + y0 = y_ratio*y0; + y1 = y_ratio*y1; + for(int c=0; c= confThreshold) { int obj_class = c; float prob = dets[j].prob[c]; - // convert to image coords - x0 = x_ratio*x0; - x1 = x_ratio*x1; - y0 = y_ratio*y0; - y1 = y_ratio*y1; - tk::dnn::box res; res.cl = obj_class; res.prob = prob; -- 2.52.0 From f778e1aa998f894654b24c0ab9ad759c0eb14019 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Wed, 5 Aug 2020 19:55:10 +0200 Subject: [PATCH 079/228] Fixed boxes to float, add conf thresh as param Signed-off-by: Micaela Verucchi --- README.md | 3 ++- demo/config.yaml | 2 +- demo/demo/demo.cpp | 5 ++++- demo/demo/map.cpp | 2 +- include/tkDNN/CenternetDetection.h | 2 +- include/tkDNN/DetectionNN.h | 2 +- include/tkDNN/MobilenetDetection.h | 2 +- include/tkDNN/Yolo3Detection.h | 2 +- src/CenternetDetection.cpp | 3 ++- src/MobilenetDetection.cpp | 3 ++- src/Yolo3Detection.cpp | 11 ++++++----- 11 files changed, 22 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a1b5b16..d9b5755 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ Once you have succesfully created your rt file, run the demo: ``` ./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y ``` -In general the demo program takes 6 parameters: +In general the demo program takes 7 parameters: ``` ./demo ``` @@ -204,6 +204,7 @@ where * ``````is the number of classes the network is trained on * `````` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network). * `````` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1) +* `````` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed. N.b. By default it is used FP32 inference diff --git a/demo/config.yaml b/demo/config.yaml index e6f91a7..31ac599 100644 --- a/demo/config.yaml +++ b/demo/config.yaml @@ -3,5 +3,5 @@ map_points : 101 #number of recall points (0 for all, 101 for COCO, 11 Pascal map_levels : 10 #number of IoU step for the AP map_step : 0.05 #step of IoU IoU_thresh : 0.5 #starting IoU threshold -conf_thresh : 0.0 #threshold on the condifence of the bbox +conf_thresh : 0.001 #threshold on the condifence of the bbox verbose : false #print on screen information diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 76b451d..9f50d0b 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -40,6 +40,9 @@ int main(int argc, char *argv[]) { bool show = true; if(argc > 6) show = atoi(argv[6]); + float conf_thresh=0.3; + if(argc > 7) + conf_thresh = atof(argv[7]); if(n_batch < 1 || n_batch > 64) FatalError("Batch dim not supported"); @@ -69,7 +72,7 @@ int main(int argc, char *argv[]) { FatalError("Network type not allowed (3rd parameter)\n"); } - detNN->init(net, n_classes, n_batch); + detNN->init(net, n_classes, n_batch, conf_thresh); gRun = true; diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index d724db0..356e35a 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -105,7 +105,7 @@ int main(int argc, char *argv[]) default: FatalError("Network type not allowed (3rd parameter)\n"); } - detNN->init(net, n_classes); + detNN->init(net, n_classes, 1, conf_thresh); //read images std::ifstream all_labels(labels_path); diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 227cb78..3c8cfbb 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -73,7 +73,7 @@ public: CenternetDetection() {}; ~CenternetDetection() {}; - bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1); + bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index 030cf8f..ba42834 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -84,7 +84,7 @@ class DetectionNN { * @param n_batches maximum number of batches to use in inference * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1) = 0; + virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0; /** * This method performs the whole detection of the NN. diff --git a/include/tkDNN/MobilenetDetection.h b/include/tkDNN/MobilenetDetection.h index cabd7eb..9a5fedc 100644 --- a/include/tkDNN/MobilenetDetection.h +++ b/include/tkDNN/MobilenetDetection.h @@ -65,7 +65,7 @@ public: MobilenetDetection() {}; ~MobilenetDetection() {}; - bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1); + bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/Yolo3Detection.h b/include/tkDNN/Yolo3Detection.h index 6d38514..100a720 100644 --- a/include/tkDNN/Yolo3Detection.h +++ b/include/tkDNN/Yolo3Detection.h @@ -24,7 +24,7 @@ public: Yolo3Detection() {}; ~Yolo3Detection() {}; - bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1); + bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 9d8df38..394e24a 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -3,11 +3,12 @@ namespace tk { namespace dnn { -bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){ +bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; nBatches = n_batches; + confThreshold = conf_thresh; dim = netRT->input_dim; diff --git a/src/MobilenetDetection.cpp b/src/MobilenetDetection.cpp index c905fea..3c54e28 100644 --- a/src/MobilenetDetection.cpp +++ b/src/MobilenetDetection.cpp @@ -126,12 +126,13 @@ float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){ return iou; } -bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){ +bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); imageSize = netRT->input_dim.h; classes = n_classes; nBatches = n_batches; + confThreshold = conf_thresh; SSDSpec specs[N_SSDSPEC]; diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index 606c6d1..e9b0064 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -3,13 +3,14 @@ namespace tk { namespace dnn { -bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches) { +bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { //convert network to tensorRT std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); nBatches = n_batches; + confThreshold = conf_thresh; tk::dnn::dataDim_t idim = netRT->input_dim; idim.n = nBatches; @@ -109,10 +110,10 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ detected.clear(); for(int j=0; j Date: Thu, 6 Aug 2020 11:05:26 +0200 Subject: [PATCH 080/228] Update README.md --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d9b5755..ea55785 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capoton "A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020) ``` -## Results +## FPS Results Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesion as the input size, on * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 ); @@ -40,6 +40,20 @@ Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesio | Nano | yolo4 512 | 2,32 | 2,34 | 3,02 | 3,04 | - | - | | Nano | yolo4 608 | 1,40 | 1,41 | 1,92 | 1,93 | - | - | +## MAP Results +Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001 + +| | CodaLab | CodaLab | CodaLab | CodaLab | tkDNN map | tkDNN map | +| -------------------- | :-----------: | :-------: | :-----------: | :---------: | :-----------: | :-------: | +| | **tkDNN** | **tkDNN** | **darknet** | **darknet** | **tkDNN** | **tkDNN** | +| | MAP(0.5:0.95) | AP50 | MAP(0.5:0.95) | AP50 | MAP(0.5:0.95) | AP50 | +| Yolov3 (416x416) | 0.381 | 0.675 | 0.380 | 0.675 | 0.372 | 0.663 | +| yolov4 (416x416) | 0.468 | 0.705 | 0.471 | 0.710 | 0.459 | 0.695 | +| yolov3tiny (416x416) | 0.096 | 0.202 | 0.096 | 0.201 | 0.093 | 0.198 | +| yolov4tiny (416x416) | 0.202 | 0.400 | 0.201 | 0.400 | 0.197 | 0.395 | +| Cnet-dla34 (512x512) | 0.366 | 0.543 | \- | \- | 0.361 | 0.535 | +| mv2SSD (512x512) | 0.226 | 0.381 | \- | \- | 0.223 | 0.378 | + ## Index - [tkDNN](#tkdnn) - [Index](#index) -- 2.52.0 From df5443e017f9390b3f282a5c18ecf335ffafc5f8 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 6 Aug 2020 15:53:57 +0200 Subject: [PATCH 081/228] Fix boxes also for Centernet Signed-off-by: Micaela Verucchi --- src/CenternetDetection.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 394e24a..46757f4 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -372,10 +372,10 @@ void CenternetDetection::postprocess(const int bi, const bool mAP){ // std::cout<<"th: "< Date: Fri, 11 Sep 2020 09:13:59 +0200 Subject: [PATCH 082/228] Fix typos (#107) Signed-off-by: micaela --- README.md | 14 +++++++------- include/tkDNN/DetectionNN.h | 8 ++++---- include/tkDNN/ImuOdom.h | 4 ++-- include/tkDNN/Layer.h | 20 ++++++++++---------- include/tkDNN/Network.h | 8 ++++---- include/tkDNN/NetworkRT.h | 2 +- include/tkDNN/evaluation.h | 8 ++++---- include/tkDNN/pluginsRT/DeformableConvRT.h | 2 +- include/tkDNN/test.h | 2 +- src/DarknetParser.cpp | 2 +- src/DeformConv2d.cpp | 2 +- src/Dense.cpp | 2 +- src/LSTM.cpp | 10 +++++----- src/LayerWgs.cpp | 2 +- src/MulAdd.cpp | 2 +- src/NetworkRT.cpp | 2 +- src/Region.cpp | 2 +- src/Shortcut.cpp | 2 +- src/evaluation.cpp | 6 +++--- 19 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index ea55785..f17f75e 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capoton ``` ## FPS Results -Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesion as the input size, on +Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 ); * Tx2, Jetpack 4.2 (CUDA 10.0, CUDNN 7.3.1, tensorrt 5.0.6 ); @@ -169,7 +169,7 @@ tkDNN implement and easy parser for darknet cfg files, a network can be converte tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names"); net->print(); ``` -All models from darknet are now parsed directly from cfg, you still need to export the weights with the descripted tools in the previus section. +All models from darknet are now parsed directly from cfg, you still need to export the weights with the described tools in the previous section.
Supported layers convolutional @@ -203,7 +203,7 @@ cmake .. -DDEBUG=True make ``` -Once you have succesfully created your rt file, run the demo: +Once you have successfully created your rt file, run the demo: ``` ./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y ``` @@ -247,7 +247,7 @@ You should provide image_list.txt and label_list.txt, using training images. How ``` bash scripts/download_validation.sh COCO ``` -to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD insted of COCO to download BDD validation. +to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD instead of COCO to download BDD validation. Then a complete example using yolo3 and COCO dataset would be: ``` @@ -269,8 +269,8 @@ N.B. export TKDNN_BATCHSIZE=2 # build tensorRT files ``` -This will create a TensorRT file with the desidered **max** batch size. -The test will still run with a batch of 1, but the created tensorRT can manage the desidered batch size. +This will create a TensorRT file with the desired **max** batch size. +The test will still run with a batch of 1, but the created tensorRT can manage the desired batch size. ### Test batch Inference This will test the network with random input and check if the output of each batch is the same. @@ -316,7 +316,7 @@ cd build ./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml ``` -This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to subit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate). +This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to submit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate). ## Existing tests and supported networks diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index ba42834..0498d41 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -76,10 +76,10 @@ class DetectionNN { ~DetectionNN(){}; /** - * Method used to inialize the class, allocate memory and compute + * Method used to initialize the class, allocate memory and compute * needed data. * - * @param tensor_path path to the rt file og the NN. + * @param tensor_path path to the rt file of the NN. * @param n_classes number of classes for the given dataset. * @param n_batches maximum number of batches to use in inference * @return true if everything is correct, false otherwise. @@ -141,9 +141,9 @@ class DetectionNN { } /** - * Method to draw boundixg boxes and labels on a frame. + * Method to draw bounding boxes and labels on a frame. * - * @param frames orginal frame to draw bounding box on. + * @param frames original frame to draw bounding box on. */ void draw(std::vector& frames) { tk::dnn::box b; diff --git a/include/tkDNN/ImuOdom.h b/include/tkDNN/ImuOdom.h index 58def96..6d8d4cb 100644 --- a/include/tkDNN/ImuOdom.h +++ b/include/tkDNN/ImuOdom.h @@ -44,7 +44,7 @@ class ImuOdom { virtual ~ImuOdom() {} /** - * Method used for inizialize the class + * Method used for initialize the class * * @return Success of the initialization */ @@ -141,7 +141,7 @@ class ImuOdom { //odomPOS = odomPOS + deltaP.cast(); // V2 odomROT = odomROT * q.normalized().toRotationMatrix(); - // compute euler + // compute Euler auto newEULER = odomROT.eulerAngles(0, 1, 2); for(int i=0; i<3; i++) { while( fabs(newEULER(i) - odomEULER(i)) > M_PI_2 ) { diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index f2ec56d..790a431 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -171,7 +171,7 @@ public: /** - Input layer (it doesnt need weigths) + Input layer (it doesn't need weights) */ class Input : public Layer { @@ -207,7 +207,7 @@ public: /** - Avaible activation functions + Available activation functions */ typedef enum { ACTIVATION_ELU = 100, @@ -216,7 +216,7 @@ typedef enum { } tkdnnActivationMode_t; /** - Activation layer (it doesnt need weigths) + Activation layer (it doesn't need weights) */ class Activation : public Layer { @@ -318,9 +318,9 @@ public: virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); const bool bidirectional = true; /**> is the net bidir */ - bool returnSeq = false; /**> if false return only the result of last timestep */ + bool returnSeq = false; /**> if false return only the result of last timestamp */ int stateSize = 0; /**> number of hidden states */ - int seqLen = 0; /**> number of timesteps */ + int seqLen = 0; /**> number of timestamp */ int numLayers = 1; /**> number of internal layers */ protected: @@ -367,7 +367,7 @@ public: /** - Deformable Convolutionl 2d layer + Deformable Convolutional 2d layer */ class DeformConv2d : public LayerWgs { @@ -449,7 +449,7 @@ protected: /** - Avaible pooling functions (padding on tkDNN is not supported) + Available pooling functions (padding on tkDNN is not supported) */ typedef enum { POOLING_MAX = 0, @@ -460,7 +460,7 @@ typedef enum { /** Pooling layer - currenty supported only 2d pooing (also on 3d input) + currently supported only 2d pooing (also on 3d input) */ class Pooling : public Layer { @@ -526,7 +526,7 @@ public: /** Reorg layer - Mantain same dimension but change C*H*W distribution + Maintains same dimension but change C*H*W distribution */ class Reorg : public Layer { @@ -559,7 +559,7 @@ public: /** Upsample layer - Mantain same dimension but change C*H*W distribution + Maintains same dimension but change C*H*W distribution */ class Upsample : public Layer { diff --git a/include/tkDNN/Network.h b/include/tkDNN/Network.h index 2d95215..b78acff 100644 --- a/include/tkDNN/Network.h +++ b/include/tkDNN/Network.h @@ -7,12 +7,12 @@ namespace tk { namespace dnn { /** - Data rapresentation beetween layers + Data representation between layers n = batch size c = channels - h = heigth (lines) + h = height (lines) w = width (rows) - l = lenght (3rd dimension) + l = length (3rd dimension) */ struct dataDim_t { @@ -43,7 +43,7 @@ public: void releaseLayers(); /** - Do inferece for every added layer + Do inference for every added layer */ dnnType* infer(dataDim_t &dim, dnnType* data); diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index 66b4f3d..4c6c816 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -91,7 +91,7 @@ public: } /** - Do inferece + Do inference */ dnnType* infer(dataDim_t &dim, dnnType* data); void enqueue(int batchSize = 1); diff --git a/include/tkDNN/evaluation.h b/include/tkDNN/evaluation.h index 8907d9d..128eba0 100644 --- a/include/tkDNN/evaluation.h +++ b/include/tkDNN/evaluation.h @@ -73,12 +73,12 @@ double computeMap( std::vector &images,const int classes, * all the recall levels are evaluated, otherwise only * map_point recall levels are used. For COCO evaluation * 101 points are used. - * @param map_step step used to increment IoU theshold + * @param map_step step used to increment IoU threshold * @param map_levels number of IoU step to perform * @param verbose is set to true, prints on screen additional info * @param write_on_file if set to true, the results produced by this function * are written on file - * @param net name of the considerd neural network + * @param net name of the considered neural network * * @return mAP IoU_tresh:IoU_tresh+map_step*map_levels (e.g. mAP 0.5:0.95 when * map_step=0.05 and map_levels=10) @@ -89,7 +89,7 @@ double computeMapNIoULevels(std::vector &images,const int classes, const int map_levels=10, const bool verbose=false, const bool write_on_file = false, std::string net = ""); /** - * This method computes the numper of True Positive (TP), False Positive (FP), + * This method computes the number of True Positive (TP), False Positive (FP), * False Negative (FN), precision, recall and f1-score. * Those values are computer over all the detections, over all the classes. * @@ -101,7 +101,7 @@ double computeMapNIoULevels(std::vector &images,const int classes, * @param verbose is set to true, prints on screen additional info * @param write_on_file if set to true, the results produced by this function * are written on file - * @param net name of the considerd neural network + * @param net name of the considered neural network */ void computeTPFPFN( std::vector &images,const int classes, const float IoU_thresh=0.5, const float conf_thresh=0.3, diff --git a/include/tkDNN/pluginsRT/DeformableConvRT.h b/include/tkDNN/pluginsRT/DeformableConvRT.h index bff6370..225a24e 100644 --- a/include/tkDNN/pluginsRT/DeformableConvRT.h +++ b/include/tkDNN/pluginsRT/DeformableConvRT.h @@ -89,7 +89,7 @@ public: for(int b=0; b input_bins, std::vector } if(output_bins.size() != outputs.size()) { std::cout< netLayers; std::ifstream if_cfg(cfg_file); diff --git a/src/DeformConv2d.cpp b/src/DeformConv2d.cpp index b161a22..dbb71e1 100644 --- a/src/DeformConv2d.cpp +++ b/src/DeformConv2d.cpp @@ -95,7 +95,7 @@ dnnType* DeformConv2d::infer(dataDim_t &dim, dnnType* srcData) { // split conv2d outputs into offset and mask checkCuda(cudaMemcpy(offset, output_conv, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); checkCuda(cudaMemcpy(mask, output_conv + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); - // kernel sigmoide + // kernel sigmoid activationSIGMOIDForward(mask, mask, chunk_dim); // deformable convolution diff --git a/src/Dense.cpp b/src/Dense.cpp index b6a9af2..4371d06 100644 --- a/src/Dense.cpp +++ b/src/Dense.cpp @@ -37,7 +37,7 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) { // place bias into dstData checkCuda( cudaMemcpy(dstData, bias_d, dim_y*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); - //do matrix moltiplication + //do matrix multiplication checkERROR( cublasSgemv(net->cublasHandle, CUBLAS_OP_T, dim_x, dim_y, &alpha, diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 511fbee..7b87711 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -133,7 +133,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig output_dim = input_dim; output_dim.c = stateSize*(bidirectional ? 2 : 1); - // if retunseq is disabled only the last timestep is returned + // if retunseq is disabled only the last timestamp is returned if(!returnSeq) { output_dim.h = 1; output_dim.w = 1; @@ -254,7 +254,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { rnnDesc, seqLen, // number of time steps (nT) x_desc_vec_.data(), // input array of desc (nT*nC_in) - srcF, // input pointer + srcF, // input pointer hx_desc_, // initial hidden state desc hx_ptr, // initial hidden state pointer cx_desc_, // initial cell state desc @@ -281,7 +281,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { rnnDesc, seqLen, // number of time steps (nT) x_desc_vec_.data(), // input array of desc (nT*nC_in) - srcB, // input pointer + srcB, // input pointer hx_desc_, // initial hidden state desc hx_ptr, // initial hidden state pointer cx_desc_, // initial cell state desc @@ -289,7 +289,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { w_desc_, // weights desc wb_ptr, // weights pointer y_desc_vec_.data(), // output desc (nT*nC_out) - dstB_NR, // output pointer + dstB_NR, // output pointer hy_desc_, // final hidden state desc hy_ptr, // final hidden state pointer cy_desc_, // final cell state desc @@ -307,7 +307,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { one_output_dim.c*sizeof(dnnType), cudaMemcpyDeviceToDevice)); } - // if retunseq is disabled only the last timestep is returned + // if retunseq is disabled only the last timestamp is returned if(returnSeq) { // forward transpose matrixTranspose(net->cublasHandle, dstF, dstData, diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index 4afb7cc..a761327 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -105,7 +105,7 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, float2half(tmp_d, variance16_d, b_size); cudaMemcpy(variance16_h, variance16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); - //conver scales + //convert scales float2half(scales_d, scales16_d, b_size); cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); diff --git a/src/MulAdd.cpp b/src/MulAdd.cpp index 0c2a962..25cec8d 100644 --- a/src/MulAdd.cpp +++ b/src/MulAdd.cpp @@ -12,7 +12,7 @@ MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) { int size = input_dim.tot(); - // create a vector with all value setted to add + // create a vector with all value set to add dnnType *add_vector_h = new dnnType[size]; for(int i=0; igetBindingIndex("data"); buf_output_idx = engineRT->getBindingIndex("out"); - std::cout<<"input idex = "< output index = "< output index = "<getBindingDimensions(buf_input_idx); diff --git a/src/Region.cpp b/src/Region.cpp index 65bb786..7c26208 100644 --- a/src/Region.cpp +++ b/src/Region.cpp @@ -63,7 +63,7 @@ dnnType* Region::infer(dataDim_t &dim, dnnType* srcData) { } -/* Intepret class */ +/* Interpret class */ RegionInterpret::RegionInterpret(dataDim_t input_dim, dataDim_t output_dim, int classes, int coords, int num, float thresh, std::string fname_weights) { diff --git a/src/Shortcut.cpp b/src/Shortcut.cpp index 78a2f23..2c7a4f4 100644 --- a/src/Shortcut.cpp +++ b/src/Shortcut.cpp @@ -13,7 +13,7 @@ Shortcut::Shortcut(Network *net, Layer *backLayer) : Layer(net) { if( /*backLayer->output_dim.c != input_dim.c ||*/ backLayer->output_dim.w != input_dim.w || backLayer->output_dim.h != input_dim.h ) - FatalError("Shortcut dim missmatch"); + FatalError("Shortcut dim mismatch"); } Shortcut::~Shortcut() { diff --git a/src/evaluation.cpp b/src/evaluation.cpp index 58c951d..f23c380 100644 --- a/src/evaluation.cpp +++ b/src/evaluation.cpp @@ -63,7 +63,7 @@ double computeMap( std::vector &images,const int classes, int gt_checked = 0; - // for each detection comput IoU with groundtruth and match detetcion and + // for each detection compute IoU with groundtruth and match detetcion and // groundtruth with IoU greater than IoU_thresh for(auto &img:images){ for(size_t i=0; i &images,const int classes, } } - //compute average precision for each class. Two methods are avaible, + //compute average precision for each class. Two methods are available, //based on map_points required double mean_average_precision = 0; double last_recall, last_precision, delta_recall; @@ -287,7 +287,7 @@ void computeTPFPFN( std::vector &images,const int classes, } } - //count all TP, FP, FN and compute precsion, recall and f1-score + //count all TP, FP, FN and compute precision, recall and f1-score double avg_precision = 0, avg_recall = 0, f1_score = 0; int TP = 0, FP = 0, FN = 0; for(size_t i=0; i Date: Tue, 15 Sep 2020 10:50:24 +0200 Subject: [PATCH 083/228] Update README with Xavier NX FPS results Signed-off-by: Micaela Verucchi --- README.md | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f17f75e..54b8140 100644 --- a/README.md +++ b/README.md @@ -18,27 +18,32 @@ M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capoton Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 ); + * Xavier NX, Jetpack 4.4 (CUDA 10.2, CUDNN 8.0.0, tensorrt 7.1.0 ). * Tx2, Jetpack 4.2 (CUDA 10.0, CUDNN 7.3.1, tensorrt 5.0.6 ); * Jetson Nano, Jetpack 4.4 (CUDA 10.2, CUDNN 8.0.0, tensorrt 7.1.0 ). | Platform | Network | FP32, B=1 | FP32, B=4 | FP16, B=1 | FP16, B=4 | INT8, B=1 | INT8, B=4 | | :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | -| RTX 2080Ti | yolo4 320 | 118,59 |237,31 | 207,81 | 443,32 | 262,37 | 530,93 | -| RTX 2080Ti | yolo4 416 | 104,81 |162,86 | 169,06 | 293,78 | 206,93 | 353,26 | -| RTX 2080Ti | yolo4 512 | 92,98 |132,43 | 140,36 | 215,17 | 165,35 | 254,96 | -| RTX 2080Ti | yolo4 608 | 63,77 |81,53 | 111,39 | 152,89 | 127,79 | 184,72 | -| AGX Xavier | yolo4 320 | 26,78 |32,05 | 57,14 | 79,05 | 73,15 | 97,56 | -| AGX Xavier | yolo4 416 | 19,96 |21,52 | 41,01 | 49,00 | 50,81 | 60,61 | -| AGX Xavier | yolo4 512 | 16,58 |16,98 | 31,12 | 33,84 | 37,82 | 41,28 | -| AGX Xavier | yolo4 608 | 9,45 |10,13 | 21,92 | 23,36 | 27,05 | 28,93 | -| Tx2 | yolo4 320 | 11,18 | 12,07 | 15,32 | 16,31 | - | - | -| Tx2 | yolo4 416 | 7,30 | 7,58 | 9,45 | 9,90 | - | - | -| Tx2 | yolo4 512 | 5,96 | 5,95 | 7,22 | 7,23 | - | - | -| Tx2 | yolo4 608 | 3,63 | 3,65 | 4,67 | 4,70 | - | - | -| Nano | yolo4 320 | 4,23 | 4,55 | 6,14 | 6,53 | - | - | -| Nano | yolo4 416 | 2,88 | 3,00 | 3,90 | 4,04 | - | - | -| Nano | yolo4 512 | 2,32 | 2,34 | 3,02 | 3,04 | - | - | -| Nano | yolo4 608 | 1,40 | 1,41 | 1,92 | 1,93 | - | - | +| RTX 2080Ti | yolo4 320 | 118.59 | 237.31 | 207.81 | 443.32 | 262.37 | 530.93 | +| RTX 2080Ti | yolo4 416 | 104.81 | 162.86 | 169.06 | 293.78 | 206.93 | 353.26 | +| RTX 2080Ti | yolo4 512 | 92.98 | 132.43 | 140.36 | 215.17 | 165.35 | 254.96 | +| RTX 2080Ti | yolo4 608 | 63.77 | 81.53 | 111.39 | 152.89 | 127.79 | 184.72 | +| AGX Xavier | yolo4 320 | 26.78 | 32.05 | 57.14 | 79.05 | 73.15 | 97.56 | +| AGX Xavier | yolo4 416 | 19.96 | 21.52 | 41.01 | 49.00 | 50.81 | 60.61 | +| AGX Xavier | yolo4 512 | 16.58 | 16.98 | 31.12 | 33.84 | 37.82 | 41.28 | +| AGX Xavier | yolo4 608 | 9.45 | 10.13 | 21.92 | 23.36 | 27.05 | 28.93 | +| Xavier NX | yolo4 320 | 11.49 | 13.79 | 25.26 | 35.51 | 33.77 | 45.66 | +| Xavier NX | yolo4 416 | 8.38 | 9.65 | 18.72 | 22.97 | 23.71 | 29.28 | +| Xavier NX | yolo4 512 | 7.03 | 7.57 | 13.50 | 15.43 | 17.95 | 19.70 | +| Xavier NX | yolo4 608 | 4.49 | 4.56 | 10.10 | 11.02 | 13.09 | 14.04 | +| Tx2 | yolo4 320 | 11.18 | 12.07 | 15.32 | 16.31 | - | - | +| Tx2 | yolo4 416 | 7.30 | 7.58 | 9.45 | 9.90 | - | - | +| Tx2 | yolo4 512 | 5.96 | 5.95 | 7.22 | 7.23 | - | - | +| Tx2 | yolo4 608 | 3.63 | 3.65 | 4.67 | 4.70 | - | - | +| Nano | yolo4 320 | 4.23 | 4.55 | 6.14 | 6.53 | - | - | +| Nano | yolo4 416 | 2.88 | 3.00 | 3.90 | 4.04 | - | - | +| Nano | yolo4 512 | 2.32 | 2.34 | 3.02 | 3.04 | - | - | +| Nano | yolo4 608 | 1.40 | 1.41 | 1.92 | 1.93 | - | - | ## MAP Results Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001 -- 2.52.0 From d3372aad31d27d68593209f13e7c189752ec6e42 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 15 Sep 2020 14:09:02 +0200 Subject: [PATCH 084/228] Update README with Xavier NX FPS results 15W4Core Signed-off-by: Micaela Verucchi --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 54b8140..70ddd26 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,10 @@ Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimensi | AGX Xavier | yolo4 416 | 19.96 | 21.52 | 41.01 | 49.00 | 50.81 | 60.61 | | AGX Xavier | yolo4 512 | 16.58 | 16.98 | 31.12 | 33.84 | 37.82 | 41.28 | | AGX Xavier | yolo4 608 | 9.45 | 10.13 | 21.92 | 23.36 | 27.05 | 28.93 | -| Xavier NX | yolo4 320 | 11.49 | 13.79 | 25.26 | 35.51 | 33.77 | 45.66 | -| Xavier NX | yolo4 416 | 8.38 | 9.65 | 18.72 | 22.97 | 23.71 | 29.28 | -| Xavier NX | yolo4 512 | 7.03 | 7.57 | 13.50 | 15.43 | 17.95 | 19.70 | -| Xavier NX | yolo4 608 | 4.49 | 4.56 | 10.10 | 11.02 | 13.09 | 14.04 | +| Xavier NX | yolo4 320 | 14.56 | 16.25 | 30.14 | 41.15 | 42.13 | 53.42 | +| Xavier NX | yolo4 416 | 10.02 | 10.60 | 22.43 | 25.59 | 29.08 | 32.94 | +| Xavier NX | yolo4 512 | 8.10 | 8.32 | 15.78 | 17.13 | 20.51 | 22.46 | +| Xavier NX | yolo4 608 | 5.26 | 5.18 | 11.54 | 12.06 | 15.09 | 15.82 | | Tx2 | yolo4 320 | 11.18 | 12.07 | 15.32 | 16.31 | - | - | | Tx2 | yolo4 416 | 7.30 | 7.58 | 9.45 | 9.90 | - | - | | Tx2 | yolo4 512 | 5.96 | 5.95 | 7.22 | 7.23 | - | - | -- 2.52.0 From a0e7f05a50e5bc639a3c843139c39884d2c5a7fc Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Sat, 10 Oct 2020 13:03:01 +0200 Subject: [PATCH 085/228] Update README.md --- README.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 70ddd26..cdfa25b 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,18 @@ tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training. -If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it and micaela.verucchi@unimore.it or refer to https://hipert.unimore.it/ . +If you use tkDNN in your research, please cite the [following paper](https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9212130&casa_token=sQTJXi7tJNoAAAAA:BguH9xCIY48MxbtDS3LXzIXzO-9sWArm7Hd7y7BwaLmqRuM_Gx8bOYizFPNMNtpo5K0kB-P-). For use in commercial solutions, write at gattifrancesco@hotmail.it and micaela.verucchi@unimore.it or refer to https://hipert.unimore.it/ . ``` -Accepted paper @ IRC 2020, will soon be published. -M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Time clustering and LiDAR-camera fusion on embedded platforms for self-driving cars", in proceedings in IEEE Robotic Computing (2020) - -Accepted paper @ ETFA 2020, will soon be published. -M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capotondi, R. Cavicchioli, M. Bertogna, M. Solieri -"A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020) +@inproceedings{verucchi2020systematic, + title={A Systematic Assessment of Embedded Neural Networks for Object Detection}, + author={Verucchi, Micaela and Brilli, Gianluca and Sapienza, Davide and Verasani, Mattia and Arena, Marco and Gatti, Francesco and Capotondi, Alessandro and Cavicchioli, Roberto and Bertogna, Marko and Solieri, Marco}, + booktitle={2020 25th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)}, + volume={1}, + pages={937--944}, + year={2020}, + organization={IEEE} +} ``` ## FPS Results -- 2.52.0 From 86478f9384eef13d68a9406ee12fbcb4df6ab892 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Fri, 23 Oct 2020 11:40:55 +0200 Subject: [PATCH 086/228] Add yolo4_mmr test Signed-off-by: Micaela Verucchi --- tests/darknet/cfg/yolo4_mmr.cfg | 1158 +++++++++++++++++++++++++++++++ tests/darknet/names/mmr.names | 4 + tests/darknet/yolo4_mmr.cpp | 34 + 3 files changed, 1196 insertions(+) create mode 100644 tests/darknet/cfg/yolo4_mmr.cfg create mode 100644 tests/darknet/names/mmr.names create mode 100644 tests/darknet/yolo4_mmr.cpp diff --git a/tests/darknet/cfg/yolo4_mmr.cfg b/tests/darknet/cfg/yolo4_mmr.cfg new file mode 100644 index 0000000..90a7204 --- /dev/null +++ b/tests/darknet/cfg/yolo4_mmr.cfg @@ -0,0 +1,1158 @@ +[net] +batch=1 +subdivisions=1 +# Training +width=512 +height=512 +# width=608 +# height=608 +channels=3 +momentum=0.949 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.0013 +burn_in=1000 +max_batches = 16000 +policy=steps +steps=12800,14400 +scales=.1,.1 + +#cutmix=1 +mosaic=1 + +#:104x104 54:52x52 85:26x26 104:13x13 for 416 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-7 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-10 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-16 + +[convolutional] +batch_normalize=1 +filters=1024 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +### SPP ### +[maxpool] +stride=1 +size=5 + +[route] +layers=-2 + +[maxpool] +stride=1 +size=9 + +[route] +layers=-4 + +[maxpool] +stride=1 +size=13 + +[route] +layers=-1,-3,-5,-6 +### End SPP ### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = 85 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = 54 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +########################## + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=27 +activation=linear + + +[yolo] +mask = 0,1,2 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=4 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +scale_x_y = 1.2 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=256 +activation=leaky + +[route] +layers = -1, -16 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=27 +activation=linear + + +[yolo] +mask = 3,4,5 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=4 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +scale_x_y = 1.1 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=512 +activation=leaky + +[route] +layers = -1, -37 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=27 +activation=linear + + +[yolo] +mask = 6,7,8 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=4 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 +scale_x_y = 1.05 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + diff --git a/tests/darknet/names/mmr.names b/tests/darknet/names/mmr.names new file mode 100644 index 0000000..701a1fc --- /dev/null +++ b/tests/darknet/names/mmr.names @@ -0,0 +1,4 @@ +blue-cone +yellow-cone +orange-cone +big-orange-cone \ No newline at end of file diff --git a/tests/darknet/yolo4_mmr.cpp b/tests/darknet/yolo4_mmr.cpp new file mode 100644 index 0000000..85649b2 --- /dev/null +++ b/tests/darknet/yolo4_mmr.cpp @@ -0,0 +1,34 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4_mmr"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4_mmr.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/mmr.names"; + // downloadWeightsifDoNotExist(input_bins[0], bin_path, ""); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} -- 2.52.0 From 702791e41ac302ed0396034cf80d6d76303ac20a Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 23 Nov 2020 11:25:52 +0100 Subject: [PATCH 087/228] Add support for yolov4x-mish. Changes: - add parameters nms_kind, nms_thresh, new_coords to yolo layer and darknet parser - added diou nms, new method to compute the BBs - created test for yolov4x-mish called yolo4x Tested, all tests work. Problem to solve: little loss in mAP of yolo4x Signed-off-by: Micaela Verucchi --- include/tkDNN/DarknetParser.h | 3 + include/tkDNN/Layer.h | 14 +- include/tkDNN/pluginsRT/YoloRT.h | 20 +- scripts/test_all_tests.sh | 1 + src/DarknetParser.cpp | 14 +- src/NetworkRT.cpp | 12 +- src/Yolo.cpp | 68 +- src/Yolo3Detection.cpp | 7 +- tests/darknet/cfg/yolo4x.cfg | 1427 ++++++++++++++++++++++++++++++ tests/darknet/yolo4x.cpp | 36 + 10 files changed, 1571 insertions(+), 31 deletions(-) create mode 100644 tests/darknet/cfg/yolo4x.cfg create mode 100644 tests/darknet/yolo4x.cpp diff --git a/include/tkDNN/DarknetParser.h b/include/tkDNN/DarknetParser.h index 29d1e8e..089c4d6 100644 --- a/include/tkDNN/DarknetParser.h +++ b/include/tkDNN/DarknetParser.h @@ -24,7 +24,10 @@ namespace tk { namespace dnn { int num = 1; int pad = 0; int coords = 4; + int nms_kind = 0; + int new_coords= 0; float scale_xy = 1; + float nms_thresh = 0.45; std::vector layers; std::string activation = "linear"; diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 790a431..25c4565 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -610,24 +610,28 @@ public: int sort_class; }; - Yolo(Network *net, int classes, int num, std::string fname_weights,int n_masks=3, float scale_xy=1); + enum nmsKind_t {GREEDY_NMS=0, DIOU_NMS=1}; + + Yolo(Network *net, int classes, int num, std::string fname_weights,int n_masks=3, float scale_xy=1, double nms_thresh=0.45, nmsKind_t nsm_kind=GREEDY_NMS, int new_coords=0); virtual ~Yolo(); virtual layerType_t getLayerType() { return LAYER_YOLO; }; - int classes, num, n_masks; + int classes, num, n_masks, new_coords; dnnType *mask_h, *mask_d; //anchors dnnType *bias_h, *bias_d; //anchors float scaleXY; + double nms_thresh; + nmsKind_t nsm_kind; std::vector classesNames; virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); - int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh); + int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh, int new_coords=0); dnnType *predictions; - static const int MAX_DETECTIONS = 8192; + static const int MAX_DETECTIONS = 8192*2; static Yolo::detection *allocateDetections(int nboxes, int classes); - static void mergeDetections(Yolo::detection *dets, int ndets, int classes); + static void mergeDetections(Yolo::detection *dets, int ndets, int classes, double nms_thresh=0.45, nmsKind_t nsm_kind=GREEDY_NMS); }; /** diff --git a/include/tkDNN/pluginsRT/YoloRT.h b/include/tkDNN/pluginsRT/YoloRT.h index f8e596c..9af8587 100644 --- a/include/tkDNN/pluginsRT/YoloRT.h +++ b/include/tkDNN/pluginsRT/YoloRT.h @@ -8,12 +8,15 @@ class YoloRT : public IPlugin { public: - YoloRT(int classes, int num, tk::dnn::Yolo *yolo = nullptr, int n_masks=3, float scale_xy=1) { + YoloRT(int classes, int num, tk::dnn::Yolo *yolo = nullptr, int n_masks=3, float scale_xy=1, float nms_thresh=0.45, int nms_kind=0, int new_coords=0) { this->classes = classes; this->num = num; this->n_masks = n_masks; this->scaleXY = scale_xy; + this->nms_thresh = nms_thresh; + this->nms_kind = nms_kind; + this->new_coords = new_coords; mask = new dnnType[n_masks]; bias = new dnnType[num*n_masks*2]; @@ -64,7 +67,10 @@ public: for (int b = 0; b < batchSize; ++b){ for(int n = 0; n < n_masks; ++n){ int index = entry_index(b, n*w*h, 0); - activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream); + if (new_coords == 1) + activationLOGISTICForward(srcData + index, dstData + index, 4*w*h, stream); //x,y,w,h + else + activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream); //x,y if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); @@ -79,7 +85,7 @@ public: virtual size_t getSerializationSize() override { - return 6*sizeof(int) + sizeof(float)+ n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char); + return 8*sizeof(int) + 2*sizeof(float)+ n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char); } virtual void serialize(void* buffer) override { @@ -87,10 +93,13 @@ public: tk::dnn::writeBUF(buf, classes); tk::dnn::writeBUF(buf, num); tk::dnn::writeBUF(buf, n_masks); + tk::dnn::writeBUF(buf, scaleXY); + tk::dnn::writeBUF(buf, nms_thresh); + tk::dnn::writeBUF(buf, nms_kind); + tk::dnn::writeBUF(buf, new_coords); tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); - tk::dnn::writeBUF(buf, scaleXY); for(int i=0; i classesNames; dnnType *mask; diff --git a/scripts/test_all_tests.sh b/scripts/test_all_tests.sh index 770aa22..af04aff 100644 --- a/scripts/test_all_tests.sh +++ b/scripts/test_all_tests.sh @@ -73,6 +73,7 @@ do print_output $? imuodom test_net yolo4 + test_net yolo4x test_net yolo4_berkeley test_net yolo4tiny test_net yolo3 diff --git a/src/DarknetParser.cpp b/src/DarknetParser.cpp index 7d7d989..7b5410c 100644 --- a/src/DarknetParser.cpp +++ b/src/DarknetParser.cpp @@ -37,7 +37,10 @@ namespace tk { namespace dnn { std::string name,value; if(!divideNameAndValue(line, name, value)) return false; - if(name.find("width") != std::string::npos) + + if(name.find("new_coords") != std::string::npos) + fields.new_coords = std::stoi(value); + else if(name.find("width") != std::string::npos) fields.width = std::stoi(value); else if(name.find("height") != std::string::npos) fields.height = std::stoi(value); @@ -79,6 +82,13 @@ namespace tk { namespace dnn { fields.group_id = std::stoi(value); else if(name.find("scale_x_y") != std::string::npos) fields.scale_xy = std::stof(value); + else if(name.find("beta_nms") != std::string::npos) + fields.nms_thresh = std::stof(value); + else if(name.find("nms_kind") != std::string::npos){ + if(value == "greedynms") fields.nms_kind = 0; + else if(value == "diounms") fields.nms_kind = 1; + else std::cout<<"Not supported nms_kind "<classesNames = names; diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 9d38440..501ade4 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -529,7 +529,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Yolo *l) { //std::cout<<"convert Yolo\n"; //std::cout<<"New plugin YOLO\n"; - IPlugin *plugin = new YoloRT(l->classes, l->num, l, l->n_masks, l->scaleXY); + IPlugin *plugin = new YoloRT(l->classes, l->num, l, l->n_masks, l->scaleXY, l->nms_thresh, l->nsm_kind, l->new_coords); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); return lRT; @@ -739,12 +739,16 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa if(name.find("Yolo") == 0) { YoloRT *r = new YoloRT(readBUF(buf), //classes readBUF(buf), //num - nullptr, - readBUF(buf)); //n_masks + nullptr, //yolo + readBUF(buf), //n_masks + readBUF(buf), //scale_xy + readBUF(buf), //nms_thresh + readBUF(buf), //nms_kind + readBUF(buf) //new_coords + ); r->c = readBUF(buf); r->h = readBUF(buf); r->w = readBUF(buf); - r->scaleXY = readBUF(buf); for(int i=0; in_masks; i++) r->mask[i] = readBUF(buf); for(int i=0; in_masks*2*r->num; i++) diff --git a/src/Yolo.cpp b/src/Yolo.cpp index a4416be..9737e74 100644 --- a/src/Yolo.cpp +++ b/src/Yolo.cpp @@ -11,7 +11,7 @@ namespace tk { namespace dnn { -Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_masks, float scale_xy) : +Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_masks, float scale_xy, double nms_thresh, nmsKind_t nsm_kind, int new_coords) : Layer(net) { this->final = true; @@ -19,6 +19,9 @@ Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_ this->num = num; this->n_masks = n_masks; this->scaleXY = scale_xy; + this->nms_thresh = nms_thresh; + this->nsm_kind = nsm_kind; + this->new_coords = new_coords; // load anchors if(fname_weights != "") { @@ -59,12 +62,21 @@ int entry_index(int batch, int location, int entry, entry*input_dim.w*input_dim.h + loc; } -Yolo::box get_yolo_box(float *x, float *biases, int n, int index, int i, int j, int lw, int lh, int w, int h, int stride) { +Yolo::box get_yolo_box(float *x, float *biases, int n, int index, int i, int j, int lw, int lh, int w, int h, int stride, int new_coords) { Yolo::box b; - b.x = (i + x[index + 0*stride]) / lw; - b.y = (j + x[index + 1*stride]) / lh; - b.w = exp(x[index + 2*stride]) * biases[2*n] / w; - b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h; + + if(new_coords == 0){ + b.x = (i + x[index + 0*stride]) / lw; + b.y = (j + x[index + 1*stride]) / lh; + b.w = exp(x[index + 2*stride]) * biases[2*n] / w; + b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h; + } + else{ + b.x = (i + x[index + 0 * stride] * 2 - 0.5) / lw; + b.y = (j + x[index + 1 * stride] * 2 - 0.5) / lh; + b.w = x[index + 2 * stride] * x[index + 2 * stride] * 4 * biases[2 * n] / w; + b.h = x[index + 3 * stride] * x[index + 3 * stride] * 4 * biases[2 * n + 1] / h; + } return b; } @@ -75,7 +87,10 @@ dnnType* Yolo::infer(dataDim_t &dim, dnnType* srcData) { for (int b = 0; b < dim.n; ++b){ for(int n = 0; n < n_masks; ++n){ int index = entry_index(b, n*dim.w*dim.h, 0, classes, input_dim, output_dim); - activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h); + if (new_coords == 1) + activationLOGISTICForward(srcData + index, dstData + index, 4*dim.w*dim.h); + else + activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h); if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); @@ -116,7 +131,7 @@ void correct_yolo_boxes(Yolo::detection *dets, int n, int w, int h, int netw, in } } -int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh) { +int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh, int new_coords) { if(predictions == nullptr) predictions = new dnnType[output_dim.tot()]; @@ -140,7 +155,7 @@ int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int net if(objectness <= thresh) continue; int box_index = entry_index(0, n*lw*lh + i, 0, classes, input_dim, output_dim); - dets[count].bbox = get_yolo_box(predictions, bias_h, mask_h[n], box_index, col, row, lw, lh, netw, neth, lw*lh); + dets[count].bbox = get_yolo_box(predictions, bias_h, mask_h[n], box_index, col, row, lw, lh, netw, neth, lw*lh, new_coords); dets[count].objectness = objectness; dets[count].classes = classes; for(j = 0; j < classes; ++j){ @@ -193,6 +208,32 @@ float yolo_box_iou(Yolo::box a, Yolo::box b) return yolo_box_intersection(a, b)/yolo_box_union(a, b); } +void box_c(const Yolo::box a, const Yolo::box b, float& top, float& bot, float& left, float& right) { + top = std::min(a.y - a.h / 2, b.y - b.h / 2); + bot = std::max(a.y + a.h / 2, b.y + b.h / 2); + left = std::min(a.x - a.w / 2, b.x - b.w / 2); + right = std::max(a.x + a.w / 2, b.x + b.w / 2); +} + +// https://github.com/Zzh-tju/DIoU-darknet +// https://arxiv.org/abs/1911.08287 +float yolo_box_diou(const Yolo::box a, const Yolo::box b, const float nms_thresh=0.6) +{ + float top, bot, left, right; + box_c(a, b, top, bot, left, right); + float w = right - left; + float h = bot - top; + float c = w * w + h * h; + float iou = yolo_box_iou(a, b); + if (c == 0) + return iou; + + float d = (a.x - b.x) * (a.x - b.x) + (a.y - b.y) * (a.y - b.y); + float u = pow(d / c, nms_thresh); + float diou_term = u; + return iou - diou_term; +} + int yolo_nms_comparator(const void *pa, const void *pb) { Yolo::detection a = *(Yolo::detection *)pa; @@ -219,8 +260,7 @@ Yolo::detection *Yolo::allocateDetections(int nboxes, int classes) { return dets; } -void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes) { - double nms_thresh = 0.45; +void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes, double nms_thresh, nmsKind_t nsm_kind) { int total = ndets; int i, j, k; @@ -246,13 +286,13 @@ void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes) { box a = dets[i].bbox; for(j = i+1; j < total; ++j){ box b = dets[j].bbox; - if (yolo_box_iou(a, b) > nms_thresh){ + if (nsm_kind == GREEDY_NMS && yolo_box_iou(a, b) > nms_thresh) + dets[j].prob[k] = 0; + else if (nsm_kind == DIOU_NMS && yolo_box_diou(a, b, nms_thresh) > nms_thresh) dets[j].prob[k] = 0; - } } } } - } }} diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index e9b0064..b94eea9 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -32,6 +32,9 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, c memcpy(yolo[i]->bias_h, yRT->bias, sizeof(dnnType)*num*nMasks*2); yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, yRT->c, yRT->h, yRT->w); yolo[i]->classesNames = yRT->classesNames; + yolo[i]->nms_thresh = yRT->nms_thresh; + yolo[i]->nsm_kind = (tk::dnn::Yolo::nmsKind_t) yRT->nms_kind; + yolo[i]->new_coords = yRT->new_coords; } dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); @@ -102,9 +105,9 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ nDets = 0; for(int i=0; ipluginFactory->n_yolos; i++) { yolo[i]->dstData = rt_out[i]; - yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold); + yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold, yolo[i]->new_coords); } - tk::dnn::Yolo::mergeDetections(dets, nDets, classes); + tk::dnn::Yolo::mergeDetections(dets, nDets, classes, yolo[0]->nms_thresh, yolo[0]->nsm_kind); // fill detected detected.clear(); diff --git a/tests/darknet/cfg/yolo4x.cfg b/tests/darknet/cfg/yolo4x.cfg new file mode 100644 index 0000000..89f2564 --- /dev/null +++ b/tests/darknet/cfg/yolo4x.cfg @@ -0,0 +1,1427 @@ +[net] +# Testing +#batch=1 +#subdivisions=1 +# Training +batch=64 +subdivisions=8 +width=672 +height=672 +channels=3 +momentum=0.949 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.00261 +burn_in=1000 +max_batches = 500500 +policy=steps +steps=400000,450000 +scales=.1,.1 + +mosaic=1 + +letter_box=1 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=80 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=40 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=80 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=80 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=80 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=80 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=80 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=80 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=80 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=80 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=80 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=80 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-13 + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-34 + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=640 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-34 + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1280 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-19 + +[convolutional] +batch_normalize=1 +filters=1280 +size=1 +stride=1 +pad=1 +activation=mish + +########################## 6 0 6 6 3 + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=640 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +### SPP ### +[maxpool] +stride=1 +size=5 + +[route] +layers=-2 + +[maxpool] +stride=1 +size=9 + +[route] +layers=-4 + +[maxpool] +stride=1 +size=13 + +[route] +layers=-1,-3,-5,-6 +### End SPP ### + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=640 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=640 +activation=mish + +[route] +layers = -1, -15 + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 94 + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=320 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=320 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=320 +activation=mish + +[route] +layers = -1, -8 + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 57 + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=160 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=160 +activation=mish + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=160 +activation=mish + +[route] +layers = -1, -8 + +[convolutional] +batch_normalize=1 +filters=160 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=320 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 0,1,2 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.1 +objectness_smooth=0 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=4.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=320 +activation=mish + +[route] +layers = -1, -22 + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=320 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=320 +activation=mish + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=320 +activation=mish + +[route] +layers = -1,-8 + +[convolutional] +batch_normalize=1 +filters=320 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=640 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 3,4,5 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.1 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=1.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=640 +activation=mish + +[route] +layers = -1, -55 + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=640 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=640 +activation=mish + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=640 +activation=mish + +[route] +layers = -1,-8 + +[convolutional] +batch_normalize=1 +filters=640 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1280 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 6,7,8 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.1 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=0.4 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 diff --git a/tests/darknet/yolo4x.cpp b/tests/darknet/yolo4x.cpp new file mode 100644 index 0000000..b9ad003 --- /dev/null +++ b/tests/darknet/yolo4x.cpp @@ -0,0 +1,36 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4x"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer168_out.bin", + bin_path + "/debug/layer185_out.bin", + bin_path + "/debug/layer202_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4x.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/BLPpiAigZJLorQD/download"); + + + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} -- 2.52.0 From b8855b9599e52a51b371e99255063cd6f00fecd7 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 23 Nov 2020 11:34:06 +0100 Subject: [PATCH 088/228] Update README Signed-off-by: Micaela Verucchi --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cdfa25b..84e0037 100644 --- a/README.md +++ b/README.md @@ -352,7 +352,8 @@ This demo also creates a json file named ```net_name_COCO_res.json``` containing | csresnext50-panet-spp | Cross Stage Partial Network 7 | [COCO 2014](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download) | | yolo4 | Yolov4 8 | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) | | yolo4_berkeley | Yolov4 8 | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 540x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) | -| yolo4tiny | Yolov4 tiny | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | +| yolo4tiny | Yolov4 tiny 9 | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | +| yolo4x | Yolov4x-mish 9 | [COCO 2017](http://cocodataset.org/) | 80 | 672x672 | [weights](https://cloud.hipert.unimore.it/s/BLPpiAigZJLorQD/download) | ## References @@ -365,3 +366,4 @@ This demo also creates a json file named ```net_name_COCO_res.json``` containing 6. He, Kaiming, et al. "Deep residual learning for image recognition." Proceedings of the IEEE conference on computer vision and pattern recognition. 2016. 7. Wang, Chien-Yao, et al. "CSPNet: A New Backbone that can Enhance Learning Capability of CNN." arXiv preprint arXiv:1911.11929 (2019). 8. Bochkovskiy, Alexey, Chien-Yao Wang, and Hong-Yuan Mark Liao. "YOLOv4: Optimal Speed and Accuracy of Object Detection." arXiv preprint arXiv:2004.10934 (2020). +9. Bochkovskiy, Alexey, "Yolo v4, v3 and v2 for Windows and Linux" (https://github.com/AlexeyAB/darknet) -- 2.52.0 From a17e7800b9c969724e5ca1af03d3d2b2edd49964 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 23 Nov 2020 13:07:54 +0100 Subject: [PATCH 089/228] Add computation of #parameters, #MACC, and max feature map size in the tests Signed-off-by: Micaela Verucchi --- include/tkDNN/Layer.h | 4 ++++ include/tkDNN/Network.h | 1 + src/Conv2d.cpp | 5 +++++ src/DeformConv2d.cpp | 6 ++++++ src/Layer.cpp | 2 ++ src/LayerWgs.cpp | 5 ++++- src/Network.cpp | 36 ++++++++++++++++++++++++++++++++++++ 7 files changed, 58 insertions(+), 1 deletion(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 790a431..91a6af0 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -54,6 +54,10 @@ public: int id = 0; bool final; //if the layer is the final one + uint n_params = 0; + uint feature_map_size = 0; + long unsigned MACC = 0; + std::string getLayerName() { layerType_t type = getLayerType(); diff --git a/include/tkDNN/Network.h b/include/tkDNN/Network.h index b78acff..6edf248 100644 --- a/include/tkDNN/Network.h +++ b/include/tkDNN/Network.h @@ -50,6 +50,7 @@ public: bool addLayer(Layer *l); void print(); const char *getNetworkRTName(const char *network_name); + void adjustFeatureMapSizeWithShortcuts(); cudnnDataType_t dataType; cudnnTensorFormat_t tensorFormat; diff --git a/src/Conv2d.cpp b/src/Conv2d.cpp index b57cf58..2901a70 100644 --- a/src/Conv2d.cpp +++ b/src/Conv2d.cpp @@ -166,6 +166,11 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, } initCUDNN(deConv); + if(this->groups != 1) + MACC = kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h; + else + MACC = input_dim.c*kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h; + // allocate warkspace if (ws_sizeInBytes!=0) { checkCuda( cudaMalloc(&workSpace, ws_sizeInBytes) ); diff --git a/src/DeformConv2d.cpp b/src/DeformConv2d.cpp index dbb71e1..826cbb0 100644 --- a/src/DeformConv2d.cpp +++ b/src/DeformConv2d.cpp @@ -73,6 +73,12 @@ DeformConv2d::DeformConv2d( Network *net, int out_ch, int deformable_group, int output_dim.c = out_ch; initCUDNN(); + + if(this->deformableGroup != 1) + MACC = kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h; + else + MACC = input_dim.c*kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h; + //allocate data for infer result checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); } diff --git a/src/Layer.cpp b/src/Layer.cpp index a355b90..7e13a48 100644 --- a/src/Layer.cpp +++ b/src/Layer.cpp @@ -18,6 +18,8 @@ Layer::Layer(Network *net) { if(!net->addLayer(this)) FatalError("Net reached max number of layers"); } + + feature_map_size = input_dim.tot() + output_dim.tot(); } Layer::~Layer() { diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index a761327..2f875a4 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -19,6 +19,8 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, int seek = 0; readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d, seek); seek += inputs*outputs*kh*kw*kl; + n_params = seek; + this->additional_bias = additional_bias; if(additional_bias) { readBinaryFile(weights_path.c_str(), outputs, &bias2_h, &bias2_d, seek); @@ -26,15 +28,16 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, } readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek); + seek += outputs; this->batchnorm = batchnorm; if(batchnorm) { - seek += outputs; readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek); seek += outputs; readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek); seek += outputs; readBinaryFile(weights_path.c_str(), outputs, &variance_h, &variance_d, seek); + seek += outputs; float eps = TKDNN_BN_MIN_EPSILON; diff --git a/src/Network.cpp b/src/Network.cpp index 7fa291f..2c3c8c7 100644 --- a/src/Network.cpp +++ b/src/Network.cpp @@ -96,6 +96,28 @@ dataDim_t Network::getOutputDim() { return layers[num_layers-1]->output_dim; } +void Network::adjustFeatureMapSizeWithShortcuts(){ + layerType_t layer_type; + int shortcutted_idx; + + for(int i=0; igetLayerType(); + if(layer_type == LAYER_SHORTCUT){ + shortcutted_idx = -1; + for(int j=0; j(layers[i])->backLayer == layers[j]){ + shortcutted_idx = j; + break; + } + } + if(shortcutted_idx == -1) + FatalError("Problem when computing featuer_map_size with shortcuts"); + for(int j=shortcutted_idx+1; jfeature_map_size += layers[shortcutted_idx]->output_dim.tot(); + } + } +} + void Network::print() { printCenteredTitle(" NETWORK MODEL ", '=', 60); @@ -106,10 +128,21 @@ void Network::print() { std::cout.width(16); std::cout<input_dim; dataDim_t out = layers[i]->output_dim; + tot_params += layers[i]->n_params; + tot_MACC += layers[i]->MACC; + if(layers[i]->feature_map_size> max_feature_map_size) + max_feature_map_size = layers[i]->feature_map_size; + std::cout.width(3); std::cout<getLayerName(); @@ -128,6 +161,9 @@ void Network::print() { } printCenteredTitle("", '=', 60); std::cout<<"\n"; + std::cout<<"N params: "< Date: Tue, 24 Nov 2020 12:37:33 +0100 Subject: [PATCH 090/228] Add shelfnet_mapillary, README_seg, resize of input Signed-off-by: Micaela Verucchi --- CMakeLists.txt | 3 + README_seg.md | 56 +++++ demo/demo/seg_demo.cpp | 48 ++++- include/tkDNN/NetworkViz.h | 4 +- include/tkDNN/SegmentationNN.h | 8 +- src/NetworkViz.cpp | 108 +++++++++- tests/shelfnet/shelfnet_mapillary.cpp | 295 ++++++++++++++++++++++++++ 7 files changed, 498 insertions(+), 24 deletions(-) create mode 100644 README_seg.md create mode 100644 tests/shelfnet/shelfnet_mapillary.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e9c27c..c292300 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,6 +110,9 @@ target_link_libraries(test_shelfnet tkDNN) add_executable(test_shelfnet_berkeley tests/shelfnet/shelfnet_berkeley.cpp) target_link_libraries(test_shelfnet_berkeley tkDNN) +add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp) +target_link_libraries(test_shelfnet_mapillary tkDNN) + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) diff --git a/README_seg.md b/README_seg.md new file mode 100644 index 0000000..a9afcd0 --- /dev/null +++ b/README_seg.md @@ -0,0 +1,56 @@ +# Semantic Segmentation with tkDNN + +Currently tkDNN supports only ShelfNet as semantic segmentation network. + +## Export weights from Shelfnet +To get the weights needed to run Mobilenet tests use [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of a Pytorch implementation of Shelfnet network. + +``` +git clone https://git.hipert.unimore.it/mverucchi/shelfnet +cd shelfnet +cd ShelfNet18_realtime +conda env create --file shelfnet_env.yml +conda activate shelfnet +mkdir layer debug +python export.py +``` + + +## Run the demo + +To run the semantic segmentation demo follow these steps (example with shelfnet_mapillary): +``` +rm shelfnet_mapillary_fp32.rt # be sure to delete(or move) old tensorRT files +export TKDNN_BATCHSIZE=4 # be sure you have batch size > than 1 if you want to run inference on images bigger than 1024 +./test_shelfnet_mapillary # run the yolo test (is slow) +./demo shelfnet_mapillary_fp32.rt ../demo/yolo_test.mp4 1 15 +``` +In general the demo program takes the following parameters: +``` +./seg_demo +``` +where +* `````` is the rt file generated by a test +* ```<``` is the path to a video file or a camera input +* `````` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network). +* ``````is the number of classes the network is trained on +* `````` if set to 0 the demo will not resize the input frames, but use it as it is, otherwise it will resize it. +* `````` is `````` is set to 1, then the input frames will be proportionally resized using `````` as width baseline. +* `````` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1) +* `````` if set to 0 (deafult) the demo will run, otherwise the evaluation of a dataset will run and the output of the segmentation will be saved. Attention: this is under development and paths are embedded, so change them in the code in advance. + +N.b. By default it is used FP32 inference + + + +## Existing tests and supported networks + +| Test Name | Network | Dataset | N Classes | Input size | Weights | +| :---------------- | :-------------------------------------------- | :-----------------------------------------------------------: | :-------: | :-----------: | :------------------------------------------------------------------------ | +| shelfnet | ShelfNet18_realtime1 | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) | +| shelfnet_berkeley | ShelfNet18_realtime1 | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) | +| shelfnet_mapillary | ShelfNet18_realtime1 | [Mapillary Vistas](https://www.mapillary.com/dataset/vistas?pKey=aFWuj_m4nGoq3-tDz5KAqQ)* | 15 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/6WnZCKLjik7xrny/download) | + +1. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019. + +*. Mapillary Vistas has originally 66 classes, but we reduced them to 15 to improve the results on the categories of our interest. \ No newline at end of file diff --git a/demo/demo/seg_demo.cpp b/demo/demo/seg_demo.cpp index 35b4fa8..bd31a1a 100644 --- a/demo/demo/seg_demo.cpp +++ b/demo/demo/seg_demo.cpp @@ -48,22 +48,37 @@ int main(int argc, char *argv[]) { int n_classes = 19; if(argc > 4) n_classes = atoi(argv[4]); - bool show = true; + bool resize = false; if(argc > 5) - show = atoi(argv[5]); - bool write_pred = false; + resize = atoi(argv[5]); + int baseline_resize = 1024; if(argc > 6) - write_pred = atoi(argv[6]); - + baseline_resize = atoi(argv[6]); + bool show = true; + if(argc > 7) + show = atoi(argv[7]); + bool write_pred = false; + if(argc > 8) + write_pred = atoi(argv[8]); + if(resize && (baseline_resize < 0 || baseline_resize > 5000)) + FatalError("Problem with baseline resize") if(n_batch < 1 || n_batch > 64) FatalError("Batch dim not supported"); + std::string net_name; + removePathAndExtension(net, net_name); + bool mapillary_15 = false; //TODO change me pls + if(n_classes == 15 && net_name == "shelfnet_mapillary_fp32") + mapillary_15 = true; + + //net initialization tk::dnn::SegmentationNN segNN; segNN.init(net, n_classes, n_batch); int height = 0, width = 0; - + int basewidth=baseline_resize, hsize; + if(write_pred){ std::string gt_folder = "../demo/CityScapes_val/images/"; std::string images_names = "../demo/CityScapes_val/all_images.txt"; @@ -85,9 +100,16 @@ int main(int argc, char *argv[]) { cv::VideoWriter resultVideo; if(SAVE_RESULT) { - int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); - int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); - resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(1024, 1024)); + int w,h; + if(resize){ + w = basewidth; + h = int((float(cap.get(cv::CAP_PROP_FRAME_HEIGHT))*float(basewidth/float(cap.get(cv::CAP_PROP_FRAME_WIDTH))))); + } + else{ + w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + } + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); } cv::Mat frame; @@ -95,11 +117,17 @@ int main(int argc, char *argv[]) { cap >> frame; if(!frame.data) break; + + if(resize){ + hsize = int((float(frame.rows)*float(basewidth/float(frame.cols)))); + cv::resize(frame, frame, cv::Size(basewidth, hsize)); + } + height = frame.rows; width = frame.cols; //inference - segNN.updateOriginal(frame); + segNN.updateOriginal(frame, true, mapillary_15); if(show) segNN.draw(); diff --git a/include/tkDNN/NetworkViz.h b/include/tkDNN/NetworkViz.h index 94468fc..2e2c3f4 100644 --- a/include/tkDNN/NetworkViz.h +++ b/include/tkDNN/NetworkViz.h @@ -5,8 +5,8 @@ namespace tk { namespace dnn { -cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0); -cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, double min=0, double max=0); +cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, bool mapillary_15=false); +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, double min=0, double max=0, bool mapillary_15=false); cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000); }} diff --git a/include/tkDNN/SegmentationNN.h b/include/tkDNN/SegmentationNN.h index edede43..ec6ac79 100644 --- a/include/tkDNN/SegmentationNN.h +++ b/include/tkDNN/SegmentationNN.h @@ -97,7 +97,7 @@ class SegmentationNN { * * @param bi batch index */ - void postprocess(const int bi=0, bool appy_colormap = true) { + void postprocess(const int bi=0, bool appy_colormap = true, bool mapillary_15=false) { dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; dataDim_t odim = netRT->output_dim; @@ -112,7 +112,7 @@ class SegmentationNN { cv::Mat colored; if(appy_colormap) - colored = vizData2Mat(tmpOutData_h, vdim, 1024, 0, 18); + colored = vizData2Mat(tmpOutData_h, vdim, 1024, 0, classes, mapillary_15); else{ cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h); colored_fp32.convertTo(colored, CV_8UC1); @@ -234,7 +234,7 @@ class SegmentationNN { } } - void updateOriginal(cv::Mat frame, bool apply_colormap=true){ + void updateOriginal(cv::Mat frame, bool apply_colormap=true, bool mapillary_15=false){ std::vector splitted_frames; int H, W, net_H, net_W; @@ -347,7 +347,7 @@ class SegmentationNN { cv::Mat colored; if(apply_colormap) - colored = vizData2Mat(tmpOutData_h, vdim, 1024, 0, 18); + colored = vizData2Mat(tmpOutData_h, vdim, 1024, 0, classes, mapillary_15); else{ cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h); colored_fp32.convertTo(colored, CV_8UC1); diff --git a/src/NetworkViz.cpp b/src/NetworkViz.cpp index 842a26e..4a3f113 100644 --- a/src/NetworkViz.cpp +++ b/src/NetworkViz.cpp @@ -6,22 +6,114 @@ namespace tk { namespace dnn { -cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max) { +cv::Mat mapillary_15_map(cv::Mat adjMap){ + + // cv::imshow("test", adjMap); + // cv::waitKey(0); + cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1); + + M3.at(0)=165; + M2.at(0)=42; + M1.at(0)=45; + + M3.at(1)=196; + M2.at(1)=196; + M1.at(1)=196; + + M3.at(2)=90; + M2.at(2)=120; + M1.at(2)=150; + + M3.at(3)=128; + M2.at(3)=64; + M1.at(3)=128; + + M3.at(4)=70; + M2.at(4)=70; + M1.at(4)=70; + + M3.at(5)=220; + M2.at(5)=20; + M1.at(5)=60; + + M3.at(6)=255; + M2.at(6)=255; + M1.at(6)=255; + + M3.at(7)=107; + M2.at(7)=142; + M1.at(7)=35; + + M3.at(8)=70; + M2.at(8)=130; + M1.at(8)=180; + + M3.at(9)=220; + M2.at(9)=220; + M1.at(9)=220; + + M3.at(10)=153; + M2.at(10)=153; + M1.at(10)=153; + + M3.at(11)=128; + M2.at(11)=128; + M1.at(11)=128; + + M3.at(12)=119; + M2.at(12)=11; + M1.at(12)=32; + + M3.at(13)=0; + M2.at(13)=0; + M1.at(13)=142; + + for(int i=14;i<256;i++) + { + M1.at(i)=0; + M2.at(i)=0; + M3.at(i)=0; + } + + cv::Mat r1,r2,r3; + + cv::LUT(adjMap,M1,r1); + cv::LUT(adjMap,M2,r2); + cv::LUT(adjMap,M3,r3); + + std::vector planes; + planes.push_back(r1); + planes.push_back(r2); + planes.push_back(r3); + + cv::Mat dst; + cv::merge(planes,dst); + return dst; + + +} + +cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, bool mapillary_15) { if(min == 0 && max == 0) cv::minMaxIdx(map, &min, &max); cv::Mat adjMap; - // expand your range to 0..255. Similar to histEq(); - map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); - //return adjMap; - cv::Mat falseColorsMap; - applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_VIRIDIS); + + if(mapillary_15){ + map.convertTo(adjMap,CV_8UC1); + falseColorsMap = mapillary_15_map(adjMap); + } + else{ + // expand your range to 0..255. Similar to histEq(); + map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); + applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_JET); + } return falseColorsMap; } -cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, double min, double max) { +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, double min, double max, bool mapillary_15) { dnnType *data = nullptr; // copy to CPU @@ -37,7 +129,7 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, doub cv::Mat grid = cv::Mat(gridSize, CV_8UC3, cv::Scalar(0)); for(int i=0; i +#include +#include + +#include "tkdnn.h" +#include "NetworkViz.h" + + +const char *input_bin = "shelfnet_mapillary/debug/input.bin"; + +const char *backbone[] = { + "shelfnet_mapillary/layers/backbone-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer1-0-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer1-0-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer1-1-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer1-1-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer2-0-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer2-0-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer2-0-downsample-0.bin", + "shelfnet_mapillary/layers/backbone-layer2-1-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer2-1-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer3-0-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer3-0-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer3-0-downsample-0.bin", + "shelfnet_mapillary/layers/backbone-layer3-1-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer3-1-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer4-0-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer4-0-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer4-0-downsample-0.bin", + "shelfnet_mapillary/layers/backbone-layer4-1-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer4-1-conv2.bin"}; + +const char *conv_out[] = { + "shelfnet_mapillary/layers/conv_out-conv-conv.bin", + "shelfnet_mapillary/layers/conv_out-conv_out.bin", + "shelfnet_mapillary/layers/conv_out16-conv-conv.bin", + "shelfnet_mapillary/layers/conv_out16-conv_out.bin", + "shelfnet_mapillary/layers/conv_out32-conv-conv.bin", + "shelfnet_mapillary/layers/conv_out32-conv_out.bin" + }; + +const char *decoder[] = { + "shelfnet_mapillary/layers/decoder-bottom-conv1.bin", + "shelfnet_mapillary/layers/decoder-bottom-conv12.bin", + "shelfnet_mapillary/layers/decoder-up_conv_list-0-conv-conv.bin", + "shelfnet_mapillary/layers/decoder-up_conv_list-0-conv_atten.bin", + "shelfnet_mapillary/layers/decoder-up_dense_list-0-conv.bin", + "shelfnet_mapillary/layers/decoder-up_conv_list-1-conv-conv.bin", + "shelfnet_mapillary/layers/decoder-up_conv_list-1-conv_atten.bin", + "shelfnet_mapillary/layers/decoder-up_dense_list-1-conv.bin" + }; + + +const char *ladder[] = { + "shelfnet_mapillary/layers/ladder-inconv-conv1.bin", + "shelfnet_mapillary/layers/ladder-inconv-conv12.bin", + "shelfnet_mapillary/layers/ladder-down_module_list-0-conv1.bin", + "shelfnet_mapillary/layers/ladder-down_module_list-0-conv12.bin", + "shelfnet_mapillary/layers/ladder-down_conv_list-0.bin", + + "shelfnet_mapillary/layers/ladder-down_module_list-1-conv1.bin", + "shelfnet_mapillary/layers/ladder-down_module_list-1-conv12.bin", + "shelfnet_mapillary/layers/ladder-down_conv_list-1.bin", + + "shelfnet_mapillary/layers/ladder-bottom-conv1.bin", + "shelfnet_mapillary/layers/ladder-bottom-conv12.bin", + + + + "shelfnet_mapillary/layers/ladder-up_conv_list-0-conv-conv.bin", + "shelfnet_mapillary/layers/ladder-up_conv_list-0-conv_atten.bin", + "shelfnet_mapillary/layers/ladder-up_dense_list-0-conv.bin", + + + "shelfnet_mapillary/layers/ladder-up_conv_list-1-conv-conv.bin", + "shelfnet_mapillary/layers/ladder-up_conv_list-1-conv_atten.bin", + "shelfnet_mapillary/layers/ladder-up_dense_list-1-conv.bin"}; + +const char *trans[] = { + "shelfnet_mapillary/layers/trans1-conv.bin", + "shelfnet_mapillary/layers/trans2-conv.bin", + "shelfnet_mapillary/layers/trans3-conv.bin"}; +int main() +{ + + downloadWeightsifDoNotExist(input_bin, "shelfnet_mapillary", "https://cloud.hipert.unimore.it/s/6WnZCKLjik7xrny/download"); + + int classes = 15; + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1); + tk::dnn::Network net(dim); + + int bi = 0, di = 0, li = 0, ci = 0; + new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); + + + + for(int i=0; i<2; ++i){ + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + } + + std::vector features; + for(int i=0;i<3;++i){ + int out_channel = pow(2,7+i); + std::cout< up_out; + //bottom + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + std::cout<output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, features[1-i]); + + //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + //LADDER + + std::vector down_out; + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + + for(int i=0; i<2;++i){ + int out_channel = pow(2,6+i); + tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]); + + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, l_last); + l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + down_out.push_back(l_last); + + new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU + } + + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.clear(); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + + new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, down_out[1-i]); + + // //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + + // for(int i=2;i>=0;--i){ + // new tk::dnn::Route(&net, &up_out[i], 1); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, classes, 3, 3, 1, 1, 1, 1, conv_out[ci++], false); + /*up_out[i] =*/ new tk::dnn::Resize(&net, classes, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR); + // } + + new tk::dnn::Softmax(&net); + + const char *output_bin = "shelfnet_mapillary/debug/softmax.bin"; + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + std::cout<<"Input:"<output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i+out_count]; + // there is the maxpool. It isn't an output but it is necessary for the process section + if(i==0) + out_count ++; + + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} -- 2.52.0 From 48ecebe6dd187b440f1f6668de4d5634ddba9357 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 7 Dec 2020 18:45:29 +0100 Subject: [PATCH 095/228] Add CenterTrack pre, post, visualization and demo. Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 5 + include/tkDNN/CenternetDetection3DTrack.h | 176 +++++ src/CenternetDetection3DTrack.cpp | 862 ++++++++++++++++++++++ 3 files changed, 1043 insertions(+) create mode 100644 include/tkDNN/CenternetDetection3DTrack.h create mode 100644 src/CenternetDetection3DTrack.cpp diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index e838395..1a93206 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -5,6 +5,7 @@ #include #include "CenternetDetection3D.h" +#include "CenternetDetection3DTrack.h" bool gRun; bool SAVE_RESULT = false; @@ -34,6 +35,7 @@ int main(int argc, char *argv[]) { n_classes = atoi(argv[4]); tk::dnn::CenternetDetection3D cnet; + tk::dnn::CenternetDetection3DTrack ctrack; tk::dnn::DetectionNN3D *detNN; @@ -42,6 +44,9 @@ int main(int argc, char *argv[]) { case 'c': detNN = &cnet; break; + case 't': + detNN = &ctrack; + break; default: FatalError("Network type not allowed (3rd parameter)\n"); } diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h new file mode 100644 index 0000000..5149d70 --- /dev/null +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -0,0 +1,176 @@ +#ifndef CENTERNETDETECTION3DTRACK_H +#define CENTERNETDETECTION3DTRACK_H + +#include "kernels.h" +#include "utils.h" +#include "tkdnn.h" +#include +#include "opencv2/opencv.hpp" +#include +#include +#include // std::iota +#include // std::sort + +#include "DetectionNN3D.h" + +#include "kernelsThrust.h" + + +namespace tk { namespace dnn { + +struct detectionRes +{ + float score; + int cl; + cv::Mat ct, tr, bb0, bb1; + float dep; + float dim[3]; + float alpha; + float x,y,z; + float rot_y; + detectionRes() : ct(cv::Mat(cv::Size(1,2), CV_32F)), + tr(cv::Mat(cv::Size(1,2), CV_32F)), + bb0(cv::Mat(cv::Size(1,2), CV_32F)), + bb1(cv::Mat(cv::Size(1,2), CV_32F)) { } + ~detectionRes() { + ct.release(); + tr.release(); + bb0.release(); + bb1.release(); + } +}; + +struct trackingRes +{ + struct detectionRes det_res; + int tracking_id; + int age; + int active; + int color; +}; + +class CenternetDetection3DTrack : public DetectionNN3D +{ +private: + tk::dnn::dataDim_t dim; + tk::dnn::dataDim_t dim2; + tk::dnn::dataDim_t dim_hm; + tk::dnn::dataDim_t dim_wh; + tk::dnn::dataDim_t dim_reg; + tk::dnn::dataDim_t dim_track; + tk::dnn::dataDim_t dim_dep; + tk::dnn::dataDim_t dim_rot; + tk::dnn::dataDim_t dim_dim; + tk::dnn::dataDim_t dim_amodel_offset; + + /* preprocessing */ + #ifdef OPENCV_CUDACONTRIB + float *mean_d; + float *stddev_d; + #else + cv::Vec mean; + cv::Vec stddev; + dnnType *input; + #endif + float *d_ptrs; + + cv::Mat src; + cv::Mat dst; + cv::Mat dst2; + cv::Mat trans, trans2, trans_out; + + /* pre inf */ + bool iter0; + dnnType *input_pre_inf_d; + bool test_pre_inf = true; + dnnType *img_d, *hm_d; + tk::dnn::dataDim_t dim_in0; + tk::dnn::dataDim_t dim_in1; + dnnType *out_d; + + + /* postprocessing */ + int K = 100; + int width = 128;//56; // TODO + + // pointer used in the kernels + float *src_out; + int *ids_out; + + float *topk_scores; + int *topk_inds_; + float *topk_ys_; + float *topk_xs_; + int *ids_d, *ids_; + + float *ones; + + float *scores, *scores_d; + int *clses, *clses_d; + int *topk_inds_d; + float *topk_ys_d; + float *topk_xs_d; + int *inttopk_xs_d, *inttopk_ys_d; + + float *bbx0, *bby0, *bbx1, *bby1; + float *bbx0_d, *bby0_d, *bbx1_d, *bby1_d; + + int *intxs, *intys; + + float *track, *dep, *rot, *dim_, *wh, *amodel_offset; + float *track_d, *dep_d, *rot_d, *dim_d, *wh_d, *amodel_offset_d; + + float *target_coords; + + /* visualization */ + cv::Mat r; + cv::Mat calibs; + cv::Mat corners, pts3DHomo; + + std::vector> face_id; + cv::Scalar tr_colors[256]; + bool view2d = false; + + //processing + struct threshold op; + float out_thresh = 0.1; + float new_thresh = 0.3; + float vis_thresh = 0.3; + float peakThreshold = 0.2; + float centerThreshold = 0.3; //default 0.5 + + + //detections + std::vector det_res; + int count_det; + //tracks + std::vector tr_res; + int count_tr; + int track_id=0; + + + bool init_preprocessing(); + bool init_pre_inf(); + bool init_postprocessing(); + bool init_visualization(const int n_classes); + void pre_inf(); + void _get_additional_inputs(); + cv::Mat transform_preds_with_trans(float x1, float x2); + void tracking(); + +public: + tk::dnn::Network *pre_phase_net = nullptr; + CenternetDetection3DTrack() {}; + ~CenternetDetection3DTrack() {}; + bool init(const std::string& tensor_path, const int n_classes=3); + void preprocess(cv::Mat &frame); + void postprocess(); + cv::Mat draw(cv::Mat &frame); +}; + + +} // namespace dnn +} // namespace tk + + +#endif /*CENTERNETDETECTION3DTRACK_H*/ \ No newline at end of file diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp new file mode 100644 index 0000000..35488f7 --- /dev/null +++ b/src/CenternetDetection3DTrack.cpp @@ -0,0 +1,862 @@ +#include "CenternetDetection3DTrack.h" + + +namespace tk { namespace dnn { + +bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes){ + std::cout<<(tensor_path).c_str()<<"\n"; + netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); + + dim = netRT->input_dim; + dim.c = 3; + + init_preprocessing(); + init_pre_inf(); + init_postprocessing(); + init_visualization(n_classes); + + count_tr = 0; +} + +bool CenternetDetection3DTrack::init_preprocessing(){ + //image transformation + src = cv::Mat(cv::Size(2,3), CV_32F); + dst = cv::Mat(cv::Size(2,3), CV_32F); + dst2 = cv::Mat(cv::Size(2,3), CV_32F); + trans = cv::Mat(cv::Size(3,2), CV_32F); + trans2 = cv::Mat(cv::Size(3,2), CV_32F); + trans_out = cv::Mat(cv::Size(3,2), CV_32F); + + dst2.at(0,0)=width * 0.5; + dst2.at(0,1)=width * 0.5; + dst2.at(1,0)=width * 0.5; + dst2.at(1,1)=width * 0.5 + width * -0.5; + + dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); + dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); + + +#ifdef OPENCV_CUDACONTRIB + + checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) ); + checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) ); + float mean[3] = {0.40789655, 0.44719303, 0.47026116}; + float stddev[3] = {0.2886383, 0.27408165, 0.27809834}; + + checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); + checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); +#else + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot())); + mean << 0.40789655, 0.44719303, 0.47026116; + stddev << 0.2886383, 0.27408165, 0.27809834; + +#endif + + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); + checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); +} + +bool CenternetDetection3DTrack::init_pre_inf(){ + // initial steps: the first part of the network + const char *pre_img_conv1_bin = "/home/davide/Projects/repos/tkDNN/build/dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; + const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; + const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; + const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; + dim_in0 = tk::dnn::dataDim_t(1, 3, 512, 512, 1); + dim_in1 = tk::dnn::dataDim_t(1, 1, 512, 512, 1); + + + checkCuda( cudaMalloc(&out_d, netRT->input_dim.tot()*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&img_d, dim_in0.tot()*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&hm_d, dim_in1.tot()*sizeof(dnnType)) ); + // init to zeros hm + dnnType *hm_h; + checkCuda( cudaMallocHost(&hm_h, 1 * dim.h * dim.w*sizeof(dnnType)) ); + for(int i=0; i<1 * dim.h * dim.w; i++) + hm_h[i]=0.0f; + checkCuda( cudaMemcpy(hm_d, hm_h, 1 * dim.h * dim.w * sizeof(dnnType), cudaMemcpyHostToDevice) ); + checkCuda( cudaFreeHost(hm_h) ); + dnnType *i0_h, *i1_h, *i2_h; + // dnnType *i0_d, *i1_d, *i2_d; + + // const char *input_bin = "dla34_cnet3d_track/debug/input.bin"; + // const char *pre_img_bin = "dla34_cnet3d_track/debug/pre_imgages.bin"; + // const char *pre_hm_bin = "dla34_cnet3d_track/debug/pre_hms.bin"; + // readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &img_d); + // readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &hm_d); + // readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &input_pre_inf_d); + + pre_phase_net = new tk::dnn::Network(dim_in0); + //pre-img + tk::dnn::Input *in_pre_img = new tk::dnn::Input(pre_phase_net, dim_in0, img_d); + tk::dnn::Conv2d *pre_img_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true); + tk::dnn::Activation *pre_img_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + //pre-hm + tk::dnn::Input *in_pre_hm = new tk::dnn::Input(pre_phase_net, dim_in1, hm_d); + tk::dnn::Conv2d *pre_hm_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true); + tk::dnn::Activation *pre_hm_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + // image input + tk::dnn::Input *input_image = new tk::dnn::Input(pre_phase_net, dim_in0, input_pre_inf_d); + tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation *relu1 = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Shortcut *s0_input = new tk::dnn::Shortcut(pre_phase_net, pre_img_relu); + tk::dnn::Shortcut *s1_input = new tk::dnn::Shortcut(pre_phase_net, pre_hm_relu); + // output data + out_d = s1_input->dstData; + //print network model + pre_phase_net->print(); + + iter0=true; // in the first iteration the last input is equal to the current input. + return true; +} + +bool CenternetDetection3DTrack::init_postprocessing(){ + srand(0); //seed = 0 for random colors + + dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1); + dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_track = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1); + dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1); + dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1); + dim_amodel_offset = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + + checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) ); + checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); + checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); + for(int i =0; i(0,0) = 633.0; + calibs.at(0,1) = 0.0; + calibs.at(0,2) = 0.0; //w/2 + calibs.at(0,3) = 0.0; + calibs.at(1,0) = 0.0; + calibs.at(1,1) = 633.0; + calibs.at(1,2) = 0.0; //h/2 + calibs.at(1,3) = 0.0; + calibs.at(2,0) = 0.0; + calibs.at(2,1) = 0.0; + calibs.at(2,2) = 1.0; + calibs.at(2,3) = 0.0; + + // Alloc array used in the kernel + checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); +} + +bool CenternetDetection3DTrack::init_visualization(const int n_classes){ + classes = n_classes; + // const char *kitti_class_name[] = { + // "person", "car", "bicycle"}; + // classesNames = std::vector(kitti_class_name, std::end( kitti_class_name)); + + const char *class_name[] = {"car", "truck", "bus", "trailer", "construction_vehicle", "pedestrian", + "motorcycle", "bicycle", "traffic_cone", "barrier"}; + classesNames = std::vector(class_name, std::end( class_name)); + + // const char *coco_class_name[] = { + // "person", "bicycle", "car", "motorcycle", "airplane", + // "bus", "train", "truck", "boat", "traffic light", "fire hydrant", + // "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", + // "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", + // "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", + // "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", + // "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", + // "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", + // "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", + // "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", + // "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", + // "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", + // "scissors", "teddy bear", "hair drier", "toothbrush" + // }; + // classesNames = std::vector(coco_class_name, std::end( coco_class_name)); + + for(int c=0; c(0,1) = 0.0; + r.at(1,0) = 0.0; + r.at(1,1) = 1.0; + r.at(1,2) = 0.0; + r.at(2,1) = 0.0; + + corners = cv::Mat(cv::Size(8,3), CV_32F); + corners.at(1,0) = 0.0; + corners.at(1,1) = 0.0; + corners.at(1,2) = 0.0; + corners.at(1,3) = 0.0; + + pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F); + pts3DHomo.at(3,0) = 1.0; + pts3DHomo.at(3,1) = 1.0; + pts3DHomo.at(3,2) = 1.0; + pts3DHomo.at(3,3) = 1.0; + pts3DHomo.at(3,4) = 1.0; + pts3DHomo.at(3,5) = 1.0; + pts3DHomo.at(3,6) = 1.0; + pts3DHomo.at(3,7) = 1.0; + + face_id.push_back({0,1,5,4}); + face_id.push_back({1,2,6, 5}); + face_id.push_back({2,3,7,6}); + face_id.push_back({3,0,4,7}); + // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); +} + +void CenternetDetection3DTrack::_get_additional_inputs(){ + //None no additional input +} + +void CenternetDetection3DTrack::pre_inf(){ + TKDNN_TSTART + tk::dnn::dataDim_t dim_aus; + pre_phase_net->infer(dim_aus, nullptr); + TKDNN_TSTOP + checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaMemcpy(input_d, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaDeviceSynchronize() ); +} + +void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ + // -----------------------------------pre-process ------------------------------------------ + + cv::Size sz = originalSize; + cv::Size sz_old; + float scale = 1.0; + float new_height = sz.height * scale; + float new_width = sz.width * scale; + if(sz.height != sz_old.height && sz.width != sz_old.width){ + calibs.at(0,2) = new_width / 2.0f; + calibs.at(1,2) = new_height /2.0f; + float c[] = {new_width / 2.0f, new_height /2.0f}; + float s[] = {dim.w, dim.h}; + // float s = new_width >= new_height ? new_width : new_height; + // ----------- get_affine_transform + // rot_rad = pi * 0 / 100 --> 0 + dim.print(); + src.at(0,0)=c[0]; + src.at(0,1)=c[1]; + src.at(1,0)=c[0]; + src.at(1,1)=c[1] + s[0] * -0.5; + dst.at(0,0)=dim.w * 0.5; + dst.at(0,1)=dim.h * 0.5; + dst.at(1,0)=dim.w * 0.5; + dst.at(1,1)=dim.h * 0.5 + dim.w * -0.5; + + src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); + src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); + dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); + dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); + + + trans = cv::getAffineTransform( src, dst ); + trans2 = cv::getAffineTransform( dst2, src ); + trans2.convertTo(trans_out, CV_32F); + } + sz_old = sz; +#ifdef OPENCV_CUDACONTRIB + std::cout<<"OPENCV CPMTROB\n"; + cv::cuda::GpuMat im_Orig; + cv::cuda::GpuMat imageF1_d, imageF2_d; + + im_Orig = cv::cuda::GpuMat(frame); + // cv::cuda::resize (im_Orig, imageF1_d, cv::Size(new_width, new_height)); + imageF1_d = im_Orig; + checkCuda( cudaDeviceSynchronize() ); + + sz = imageF1_d.size(); + + cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR ); + checkCuda( cudaDeviceSynchronize() ); + + imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0); + checkCuda( cudaDeviceSynchronize() ); + + dim2 = dim; + cv::cuda::GpuMat bgr[3]; + cv::cuda::split(imageF1_d,bgr);//split source + + for(int i=0; i(0,0) = x1; + target_coords.at(0,1) = x2; + target_coords.at(0,2) = 1.0; + return trans_out * target_coords; +} + +void CenternetDetection3DTrack::tracking(){ + + float item_size[count_det]; + int item_cl[count_det]; + float dets[2*count_det]; + for(int i=0; i(0,0) - det_res[i].bb0.at(0,0)) * + (det_res[i].bb1.at(0,1) - det_res[i].bb0.at(0,1)); + item_cl[i] = det_res[i].cl; + dets[i*2] = det_res[i].ct.at(0,0); + dets[i*2+1] = det_res[i].ct.at(0,1); + } + + float track_size[count_tr]; + int track_cl[count_tr]; + float tracks[2*count_tr]; + for(int i=0; i(0,0) - tr_res[i].det_res.bb0.at(0,0)) * + (tr_res[i].det_res.bb1.at(0,1) - tr_res[i].det_res.bb0.at(0,1)); + track_cl[i] = tr_res[i].det_res.cl; + tracks[i*2] = tr_res[i].det_res.ct.at(0,0); + tracks[i*2+1] = tr_res[i].det_res.ct.at(0,1); + } + float dist[count_tr*count_det]; + bool invalid; + for(int i=0; i track_size[i] || dist[j*count_tr+i] > item_size[j] || item_cl[j] != track_cl[i]; + dist[j*count_tr+i] = dist[j*count_tr+i] + invalid * (1 << 18); + } + } + int matched_indices[2*count_tr]; + float min_tr; + int min_idtr=-1; + for(int i=0; i new_tr_res; + int id_new_tr=0; + for(int i=0; i new_thresh) { + count_tr_ ++; + struct trackingRes new_tr_res_; + new_tr_res_.det_res.score = det_res[i].score; + new_tr_res_.det_res.cl = det_res[i].cl; + new_tr_res_.det_res.ct = det_res[i].ct; + new_tr_res_.det_res.tr = det_res[i].tr; + new_tr_res_.det_res.bb0 = det_res[i].bb0; + new_tr_res_.det_res.bb1 = det_res[i].bb1; + new_tr_res_.det_res.dep = det_res[i].dep; + new_tr_res_.det_res.dim[0] = det_res[i].dim[0]; + new_tr_res_.det_res.dim[1] = det_res[i].dim[1]; + new_tr_res_.det_res.dim[2] = det_res[i].dim[2]; + new_tr_res_.det_res.alpha = det_res[i].alpha; + new_tr_res_.det_res.x = det_res[i].x; + new_tr_res_.det_res.y = det_res[i].y; + new_tr_res_.det_res.z = det_res[i].z; + new_tr_res_.det_res.rot_y = det_res[i].rot_y; + new_tr_res_.tracking_id = track_id++; + new_tr_res_.age = 1; + new_tr_res_.active = 1; + new_tr_res_.color = rand() % 256; + tr_res.push_back(new_tr_res_); + } + } + count_tr = count_tr_; + + if(track_id==1000) + track_id=0; + det_res.clear(); + +} + +void CenternetDetection3DTrack::postprocess(){ + dnnType *rt_out[9]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]; + rt_out[1] = (dnnType *)netRT->buffersRT[2]; + rt_out[2] = (dnnType *)netRT->buffersRT[3]; + rt_out[3] = (dnnType *)netRT->buffersRT[4]; + rt_out[4] = (dnnType *)netRT->buffersRT[5]; + rt_out[5] = (dnnType *)netRT->buffersRT[6]; + rt_out[6] = (dnnType *)netRT->buffersRT[7]; + rt_out[7] = (dnnType *)netRT->buffersRT[8]; + rt_out[8] = (dnnType *)netRT->buffersRT[9]; + + // ------------------------------------ process -------------------------------------------- + + activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot()); + checkCuda( cudaDeviceSynchronize() ); + + // output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1. + activationSIGMOIDForward(rt_out[5], rt_out[5], dim_dep.tot()); + checkCuda( cudaDeviceSynchronize() ); + transformDep(ones, ones + dim_dep.tot(), rt_out[5], rt_out[5] + dim_dep.tot()); + checkCuda( cudaDeviceSynchronize() ); + + // nms + subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op); + + // ----------- nms end + // ----------- topk + + if(K > dim_hm.h * dim_hm.w){ + printf ("Error topk (K is too large)\n"); + return; + } + + checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) ); + + sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d); + checkCuda( cudaDeviceSynchronize() ); + + topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d); + checkCuda( cudaDeviceSynchronize() ); + + checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) ); + + topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d); + checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); + + checkCuda( cudaMemcpy(intxs, inttopk_xs_d, K * sizeof(int), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(intys, inttopk_ys_d, K * sizeof(int), cudaMemcpyDeviceToHost) ); + + checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); + + // ----------- topk end + + topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out); + checkCuda( cudaDeviceSynchronize() ); + + bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out); + checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(bby0, bby0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(bbx1, bbx1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); + + //regression heads + // ['tracking', 'dep', 'rot', 'dim', 'amodel_offset', + // 'nuscenes_att', 'velocity'] + getRecordsFromTopKId(topk_inds_d, K, dim_track.c, dim_track.h * dim_track.w, rt_out[4], track_d, ids_out); + checkCuda( cudaMemcpy(track, track_d, K * dim_track.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[5], dep_d, ids_out); + checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[6], rot_d, ids_out); + checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[7], dim_d, ids_out); + checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + getRecordsFromTopKId(topk_inds_d, K, dim_amodel_offset.c, dim_amodel_offset.h * dim_amodel_offset.w, rt_out[8], amodel_offset_d, ids_out); + checkCuda( cudaMemcpy(amodel_offset, amodel_offset_d, K * dim_amodel_offset.c * sizeof(float), cudaMemcpyDeviceToHost) ); + + // ---------------------------------- post-process ----------------------------------------- + + count_det = 0; + det_res.clear(); + for(int i = 0; i(2,3); + new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs.at(0,3) - calibs.at(0,2) * new_det_res.z) / calibs.at(0,0); + new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs.at(1,3) - calibs.at(1,2) * new_det_res.z) / calibs.at(1,1) + (dim_[i] / 2); + + // alpha2rot_y + // idx = rot[:, 1] > rot[:, 5] + // alpha1 = np.arctan2(rot[:, 2], rot[:, 3]) + (-0.5 * np.pi) + // alpha2 = np.arctan2(rot[:, 6], rot[:, 7]) + ( 0.5 * np.pi) + // return alpha1 * idx + alpha2 * (1 - idx) + if(rot[1*K + i] > rot[5*K + i]) + new_det_res.alpha = std::atan2(rot[2*K + i], rot[3*K + i]) -0.5 * M_PI; + else + new_det_res.alpha = std::atan2(rot[6*K + i], rot[7*K + i]) +0.5 * M_PI; + new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at(0,0) - calibs.at(0,2), calibs.at(0,0))); + new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest + det_res.push_back(new_det_res); + + } + // track step + tracking(); +} + +cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) { + + float sc; + int id; + std::string txt; + int baseline = 0; + float font_scale = 0.8; + int thickness = 2; + for(int i=0; i vis_thresh){// && tr_res[i].active!=0) { + if(view2d) { + + + cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), tr_res[i].det_res.bb0.at(0,1)), + cv::Point(tr_res[i].det_res.bb1.at(0,0), tr_res[i].det_res.bb1.at(0,1)), tr_colors[tr_res[i].color], thickness); + cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), + tr_res[i].det_res.bb0.at(0,1) - text_size.height - thickness), + cv::Point(tr_res[i].det_res.bb0.at(0,0) + text_size.width, + tr_res[i].det_res.bb0.at(0,1)), tr_colors[tr_res[i].color], -1); + + cv::putText(frame, txt, cv::Point(tr_res[i].det_res.bb0.at(0,0), + tr_res[i].det_res.bb0.at(0,1) - thickness -1), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); + + cv::arrowedLine(frame, cv::Point((int)tr_res[i].det_res.ct.at(0,0), + (int)tr_res[i].det_res.ct.at(0,1)), + cv::Point((int)(tr_res[i].det_res.ct.at(0,0) + tr_res[i].det_res.tr.at(0,0)), + (int)(tr_res[i].det_res.ct.at(0,1) + tr_res[i].det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); + } + //3d + if(!view2d && tr_res[i].det_res.z > 1){ + r.at(0,0) = std::cos(tr_res[i].det_res.rot_y); + r.at(0,2) = std::sin(tr_res[i].det_res.rot_y); + r.at(2,0) = -std::sin(tr_res[i].det_res.rot_y); + r.at(2,2) = std::cos(tr_res[i].det_res.rot_y); + + corners.at(0,0) = tr_res[i].det_res.dim[2]/2; + corners.at(0,1) = tr_res[i].det_res.dim[2]/2; + corners.at(0,2) = -tr_res[i].det_res.dim[2]/2; + corners.at(0,3) = -tr_res[i].det_res.dim[2]/2; + corners.at(0,4) = tr_res[i].det_res.dim[2]/2; + corners.at(0,5) = tr_res[i].det_res.dim[2]/2; + corners.at(0,6) = -tr_res[i].det_res.dim[2]/2; + corners.at(0,7) = -tr_res[i].det_res.dim[2]/2; + + corners.at(1,4) = -tr_res[i].det_res.dim[0]; + corners.at(1,5) = -tr_res[i].det_res.dim[0]; + corners.at(1,6) = -tr_res[i].det_res.dim[0]; + corners.at(1,7) = -tr_res[i].det_res.dim[0]; + + corners.at(2,0) = tr_res[i].det_res.dim[1]/2; + corners.at(2,1) = -tr_res[i].det_res.dim[1]/2; + corners.at(2,2) = -tr_res[i].det_res.dim[1]/2; + corners.at(2,3) = tr_res[i].det_res.dim[1]/2; + corners.at(2,4) = tr_res[i].det_res.dim[1]/2; + corners.at(2,5) = -tr_res[i].det_res.dim[1]/2; + corners.at(2,6) = -tr_res[i].det_res.dim[1]/2; + corners.at(2,7) = tr_res[i].det_res.dim[1]/2; + + cv::Mat aus = r * corners; + + for(int k=0; k<8; k++) { + aus.at(0,k) += tr_res[i].det_res.x; + aus.at(1,k) += tr_res[i].det_res.y; + aus.at(2,k) += tr_res[i].det_res.z; + } + + // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); + for(int k1=0; k1<3; k1++) { + for(int k2=0; k2<8; k2++) + pts3DHomo.at(k1,k2) = aus.at(k1,k2); + } + + aus.release(); + aus = calibs * pts3DHomo; + std::vector res_corners; + for(int k=0; k<8; k++) { + res_corners.push_back(aus.at(0,k) / aus.at(2,k)); + res_corners.push_back(aus.at(1,k) / aus.at(2,k)); + } + aus.release(); + for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(j) * 2), + res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point(res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + tr_colors[tr_res[i].color], 2); + if(ind_f == 0) { + cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(0) * 2), + res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point(res_corners.at(face_id.at(ind_f).at(2) * 2), + res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2); + cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(1) * 2), + res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point(res_corners.at(face_id.at(ind_f).at(3) * 2), + res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2); + } + } + } + float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; + for(int k=0; k<8; k++) { + if(res_corners[2*k]bb1) + bb1=res_corners[2*k]; + if(res_corners[2*k+1]bb3) + bb3=res_corners[2*k+1]; + + } + cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), + tr_colors[tr_res[i].color], thickness); + cv::rectangle(frame, cv::Point(bb0, bb2 - text_size.height - thickness), + cv::Point(bb0 + text_size.width, bb2), tr_colors[tr_res[i].color], -1); + + cv::putText(frame, txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, + font_scale, cv::Scalar(255, 255, 255), 1); + + cv::arrowedLine(frame, cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), + cv::Point((int)((bb0 + bb1)/2 + tr_res[i].det_res.tr.at(0,0)), + (int)((bb2 + bb3)/2 + tr_res[i].det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); + } + } + + } + return frame; +} + +}} + + -- 2.52.0 From 4543df853390c5512653419905b3906733b3b378 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 7 Dec 2020 18:56:07 +0100 Subject: [PATCH 096/228] Update readme. Signed-off-by: Davide Sapienza --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 234e66b..3e4ceac 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,16 @@ cd pytorch-ssd conda env create -f env_mobv2ssd.yml python run_ssd_live_demo.py mb2-ssd-lite ``` +### 5)Export weights for CenterTrack +To get the weights needed to run CenterTrack tests use [this](https://github.com/sapienzadavide/CenterTrack.git) fork of the original CenterTrack. +``` +git clone https://github.com/sapienzadavide/CenterTrack.git +``` +* follow the instruction in the README.md and INSTALL.md + +``` +python demo.py tracking,ddd --load_model ../models/nuScenes_3Dtracking.pth --dataset nuscenes --pre_hm --track_thresh 0.1 --demo /path/to/image/or/folder/or/video/or/webcam --test_focal_length 633 --exp_wo --exp_wo_dim 512 --input_h 512 --input_w 512 +``` ## Darknet Parser tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*: @@ -246,6 +256,15 @@ The demo3D program takes the same parameters of the demo program: ./demo ``` +#### Run the 3D OD-tracking demo + +To run the 3D object detection & tracking demo follow these steps (example with CenterTrack based on DLA34): +``` +rm dla34_cnet3d_track_fp32.rt # be sure to delete(or move) old tensorRT files +./test_dla34_cnet3d_track # run the yolo test (is slow) +./demo3D dla34_cnet3d_track_fp32.rt ../demo/yolo_test.mp4 t +``` + ### FP16 inference To run the an object detection demo with FP16 inference follow these steps (example with yolov3): -- 2.52.0 From dbc052865c0c2c74994786bd62938f44d4b5a674 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 9 Dec 2020 18:04:42 +0100 Subject: [PATCH 097/228] Fix a wrong path in CenternetDetection3DTrack.cpp Signed-off-by: Davide Sapienza --- src/CenternetDetection3DTrack.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 35488f7..6b34571 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -59,7 +59,7 @@ bool CenternetDetection3DTrack::init_preprocessing(){ bool CenternetDetection3DTrack::init_pre_inf(){ // initial steps: the first part of the network - const char *pre_img_conv1_bin = "/home/davide/Projects/repos/tkDNN/build/dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; + const char *pre_img_conv1_bin = "dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; -- 2.52.0 From 9e1d7b3bb42f870b417a3229bf495e575377aed7 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 9 Dec 2020 18:05:54 +0100 Subject: [PATCH 098/228] Update demo3d with show flag Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 1a93206..609a21a 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -33,6 +33,12 @@ int main(int argc, char *argv[]) { int n_classes = 3; if(argc > 4) n_classes = atoi(argv[4]); + bool show = false; + if(argc > 5) + show = atoi(argv[5]); + + if(!show) + SAVE_RESULT = true; tk::dnn::CenternetDetection3D cnet; tk::dnn::CenternetDetection3DTrack ctrack; @@ -70,7 +76,8 @@ int main(int argc, char *argv[]) { cv::Mat frame; cv::Mat dnn_input; - cv::namedWindow("detection", cv::WINDOW_NORMAL); + if(show) + cv::namedWindow("detection", cv::WINDOW_NORMAL); std::vector detected_bbox; @@ -86,9 +93,11 @@ int main(int argc, char *argv[]) { //inference detNN->update(dnn_input); frame = detNN->draw(frame); - - cv::imshow("detection", frame); - cv::waitKey(1); + + if(show) { + cv::imshow("detection", frame); + cv::waitKey(1); + } if(SAVE_RESULT) resultVideo << frame; } -- 2.52.0 From 1cfa199ee6b301fafd707b7e0d600423b73357d0 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 9 Dec 2020 19:43:51 +0100 Subject: [PATCH 099/228] Add pre-processing and post-processing stats Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 13 +++++++++++++ include/tkDNN/DetectionNN3D.h | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 609a21a..4286faa 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -105,11 +105,24 @@ int main(int argc, char *argv[]) { std::cout<<"detection end\n"; double mean = 0; + std::cout<pre_stats.begin(), detNN->pre_stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(detNN->pre_stats.begin(), detNN->pre_stats.end())<<" ms\n"; + for(int i=0; ipre_stats.size(); i++) mean += detNN->pre_stats[i]; mean /= detNN->pre_stats.size(); + std::cout<<"Avg: "<stats.begin(), detNN->stats.end())<<" ms\n"; std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n"; for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); std::cout<<"Avg: "<post_stats.begin(), detNN->post_stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(detNN->post_stats.begin(), detNN->post_stats.end())<<" ms\n"; + for(int i=0; ipost_stats.size(); i++) mean += detNN->post_stats[i]; mean /= detNN->post_stats.size(); + std::cout<<"Avg: "< detected; /*bounding boxes in output*/ - std::vector stats; /*keeps track of inference times (ms)*/ + std::vector pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/ std::vector classesNames; DetectionNN3D() {}; @@ -107,6 +107,7 @@ class DetectionNN3D { TKDNN_TSTART preprocess(frame); TKDNN_TSTOP + pre_stats.push_back(t_ns); if(save_times) *times< Date: Mon, 14 Dec 2020 15:32:35 +0100 Subject: [PATCH 100/228] Fix a bug in the 3D bounding boxes. Signed-off-by: Davide Sapienza --- src/CenternetDetection3DTrack.cpp | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 6b34571..ef3e161 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -267,8 +267,8 @@ bool CenternetDetection3DTrack::init_visualization(const int n_classes){ face_id.push_back({0,1,5,4}); face_id.push_back({1,2,6, 5}); - face_id.push_back({2,3,7,6}); face_id.push_back({3,0,4,7}); + face_id.push_back({2,3,7,6}); // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } @@ -809,20 +809,20 @@ cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) { aus.release(); for(int ind_f = 3; ind_f>=0; ind_f--) { for(int j=0; j<4; j++) { - cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(j) * 2), - res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point(res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), + (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), tr_colors[tr_res[i].color], 2); - if(ind_f == 0) { - cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(0) * 2), - res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point(res_corners.at(face_id.at(ind_f).at(2) * 2), - res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2); - cv::line(frame, cv::Point(res_corners.at(face_id.at(ind_f).at(1) * 2), - res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point(res_corners.at(face_id.at(ind_f).at(3) * 2), - res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2); + if(ind_f == 0 && j==3) { + cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), + (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), + (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2); + cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), + (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), + (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2); } } } @@ -838,8 +838,9 @@ cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) { bb3=res_corners[2*k+1]; } - cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), - tr_colors[tr_res[i].color], thickness); + // if(not no_bbox): + // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), + // tr_colors[tr_res[i].color], thickness); cv::rectangle(frame, cv::Point(bb0, bb2 - text_size.height - thickness), cv::Point(bb0 + text_size.width, bb2), tr_colors[tr_res[i].color], -1); -- 2.52.0 From 56feb54377c0678e42077fabbf84ce2fc138f4c5 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Thu, 21 Jan 2021 00:22:15 +0400 Subject: [PATCH 101/228] able to build kernels as shared object file(dll),and minor changes to lstm.cpp and utils.cpp to overcome minor msvc build errors --- CMakeLists.txt | 11 +++++++++-- include/tkDNN/DetectionNN.h | 5 +++++ include/tkDNN/ImuOdom.h | 6 ++++++ include/tkDNN/Int8BatchStream.h | 7 ++++++- include/tkDNN/utils.h | 8 ++++++++ src/LSTM.cpp | 13 +++++++++---- src/utils.cpp | 7 ++++++- 7 files changed, 49 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c8619d..03e26c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,13 @@ cmake_minimum_required(VERSION 3.5) project (tkDNN) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) +if(LINUX) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable") +endif() +if(WIN32) +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_FLAGS "/O2 /FS ") +endif() include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN) # project specific flags @@ -18,7 +24,7 @@ add_definitions(-DTKDNN_PATH="${CMAKE_CURRENT_SOURCE_DIR}") find_package(CUDA 9.0 REQUIRED) SET(CUDA_SEPARABLE_COMPILATION ON) #set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'") -set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32) +set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32 -arch=sm_61 ) find_package(CUDNN REQUIRED) include_directories(${CUDNN_INCLUDE_DIR}) @@ -28,6 +34,7 @@ include_directories(${CUDNN_INCLUDE_DIR}) file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu") cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS}) cuda_add_library(kernels SHARED ${tkdnn_CUSRC}) +target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES}) #------------------------------------------------------------------------------- @@ -48,7 +55,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV") file(GLOB tkdnn_SRC "src/*.cpp") set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS} yaml-cpp) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES}) add_library(tkDNN SHARED ${tkdnn_SRC}) target_link_libraries(tkDNN ${tkdnn_LIBS}) diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index 0498d41..9ce33c1 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -4,7 +4,12 @@ #include #include #include +#ifdef __linux__ #include +#elif _WIN32 +#include +#endif + #include #include "utils.h" diff --git a/include/tkDNN/ImuOdom.h b/include/tkDNN/ImuOdom.h index 6d8d4cb..aace012 100644 --- a/include/tkDNN/ImuOdom.h +++ b/include/tkDNN/ImuOdom.h @@ -1,7 +1,13 @@ #include #include #include /* srand, rand */ + +#ifdef __linux__ #include +#elif _WIN32 +#include +#endif + #include #include #include "utils.h" diff --git a/include/tkDNN/Int8BatchStream.h b/include/tkDNN/Int8BatchStream.h index 4349c1f..7d2cef5 100644 --- a/include/tkDNN/Int8BatchStream.h +++ b/include/tkDNN/Int8BatchStream.h @@ -11,8 +11,13 @@ #include #include #include -#include +#include +#ifdef __linux__ #include +#elif _WIN32 +#include +#endif + #include #include "NvInfer.h" diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index 538a3f3..cb3c18f 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -12,7 +12,12 @@ #include #include +#ifdef __linux__ #include +#elif _WIN32 +#include +#endif + #include @@ -39,6 +44,7 @@ #define TKDNN_VERBOSE 0 // Simple Timer +#ifdef __linux__ #define TKDNN_TSTART timespec start, end; \ clock_gettime(CLOCK_MONOTONIC, &start); @@ -48,6 +54,8 @@ if(show) std::cout< 7 - checkCUDNN(cudnnSetRNNDescriptor_v6(net->cudnnHandle, + checkCUDNN(cudnnSetRNNDescriptor_v6(net->cudnnHandle,rnnDesc, stateSize, numLayers, dropoutDesc, + cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, + //(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), + cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL, + cudnnRNNMode_t::CUDNN_LSTM, + cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, + net->dataType)); #else - checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, -#endif - rnnDesc, stateSize, numLayers, dropoutDesc, + checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle,rnnDesc, stateSize, numLayers, dropoutDesc, cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, //(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL, cudnnRNNMode_t::CUDNN_LSTM, cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, net->dataType)); +#endif // Get temp space sizes diff --git a/src/utils.cpp b/src/utils.cpp index 65030f0..e143cce 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -170,6 +170,7 @@ void getMemUsage(double& vm_usage_kb, double& resident_set_kb){ using std::ios_base; using std::ifstream; using std::string; + SYSTEM_INFO sysInfo; vm_usage_kb = 0.0; resident_set_kb = 0.0; @@ -191,8 +192,12 @@ void getMemUsage(double& vm_usage_kb, double& resident_set_kb){ >> O >> itrealvalue >> starttime >> vsize >> rss; stat_stream.close(); - +#ifdef __linux__ long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages +#elif _WIN32 + long page_size_kb = sysInfo.dwPageSize/1024; +#endif + vm_usage_kb = vsize / 1024.0; resident_set_kb = rss * page_size_kb; } -- 2.52.0 From 512acd8cba99c7ec21d542b32a2881671da39cba Mon Sep 17 00:00:00 2001 From: perseusdg Date: Thu, 21 Jan 2021 09:29:27 +0400 Subject: [PATCH 102/228] minor fixes --- Issues.md | 1 + demo/demo/map.cpp | 5 +++++ include/tkDNN/utils.h | 14 +++++++++++--- src/Yolo.cpp | 9 +++++---- 4 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 Issues.md diff --git a/Issues.md b/Issues.md new file mode 100644 index 0000000..4875b13 --- /dev/null +++ b/Issues.md @@ -0,0 +1 @@ +1)error C2131 @ Yolo3Detection.cpp(97) -> expression doesnt evaluate to a constant caused to read of variable outside its lifetime \ No newline at end of file diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 356e35a..8e363ee 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -2,7 +2,12 @@ #include #include #include /* srand, rand */ +#ifdef __linux__ #include +#elif _WIN32 +#include +#endif + #include #include "utils.h" diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index cb3c18f..1404509 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -15,10 +15,12 @@ #ifdef __linux__ #include #elif _WIN32 -#include -#endif +#define NOMINMAX +#include +#endif #include +#include #define dnnType float @@ -55,7 +57,13 @@ #define TKDNN_TSTOP TKDNN_TSTOP_C(COL_CYANB, TKDNN_VERBOSE) #elif _WIN32 -#endif +#define TKDNN_TSTART auto start = std::chrono::high_resolution_clock::now(); +#define TKDNN_TSTOP auto stop = std::chrono::high_resolution_clock::now(); \ +std::chrono::duration duration = stop -start; \ +auto time_ms = std::chrono::duration_cast(duration);\ +double t_ns = time_ms.count(); +#endif + /******************************************************** * Prints the error message, and exits diff --git a/src/Yolo.cpp b/src/Yolo.cpp index 9737e74..6d0b546 100644 --- a/src/Yolo.cpp +++ b/src/Yolo.cpp @@ -9,6 +9,7 @@ #include "Layer.h" #include "kernels.h" + namespace tk { namespace dnn { Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_masks, float scale_xy, double nms_thresh, nmsKind_t nsm_kind, int new_coords) : @@ -209,10 +210,10 @@ float yolo_box_iou(Yolo::box a, Yolo::box b) } void box_c(const Yolo::box a, const Yolo::box b, float& top, float& bot, float& left, float& right) { - top = std::min(a.y - a.h / 2, b.y - b.h / 2); - bot = std::max(a.y + a.h / 2, b.y + b.h / 2); - left = std::min(a.x - a.w / 2, b.x - b.w / 2); - right = std::max(a.x + a.w / 2, b.x + b.w / 2); + top = (std::min)(a.y - a.h / 2, b.y - b.h / 2); + bot = (std::max)(a.y + a.h / 2, b.y + b.h / 2); + left = (std::min)(a.x - a.w / 2, b.x - b.w / 2); + right = (std::max)(a.x + a.w / 2, b.x + b.w / 2); } // https://github.com/Zzh-tju/DIoU-darknet -- 2.52.0 From adac8576b0faf515ad3f459b1f50fd16cef6d64d Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Fri, 22 Jan 2021 17:57:42 +0100 Subject: [PATCH 103/228] Add support to Scaled-YOLO4, update Yolov4x-mish (tested) Signed-off-by: Micaela Verucchi --- README.md | 4 +- include/tkDNN/Layer.h | 7 +- include/tkDNN/NetworkRT.h | 1 + .../tkDNN/pluginsRT/ActivationLogisticRT.h | 60 + include/tkDNN/pluginsRT/YoloRT.h | 14 +- scripts/test_all_tests.sh | 5 +- src/Activation.cpp | 4 + src/DarknetParser.cpp | 1 + src/NetworkRT.cpp | 13 +- src/Yolo.cpp | 21 +- tests/darknet/cfg/yolo4-csp.cfg | 1279 +++++++++++++++++ tests/darknet/cfg/yolo4x.cfg | 21 +- tests/darknet/yolo4-csp.cpp | 36 + tests/darknet/yolo4x.cpp | 2 +- 14 files changed, 1441 insertions(+), 27 deletions(-) create mode 100644 include/tkDNN/pluginsRT/ActivationLogisticRT.h create mode 100644 tests/darknet/cfg/yolo4-csp.cfg create mode 100644 tests/darknet/yolo4-csp.cpp diff --git a/README.md b/README.md index 84e0037..d9927c0 100644 --- a/README.md +++ b/README.md @@ -353,7 +353,8 @@ This demo also creates a json file named ```net_name_COCO_res.json``` containing | yolo4 | Yolov4 8 | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) | | yolo4_berkeley | Yolov4 8 | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 540x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) | | yolo4tiny | Yolov4 tiny 9 | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | -| yolo4x | Yolov4x-mish 9 | [COCO 2017](http://cocodataset.org/) | 80 | 672x672 | [weights](https://cloud.hipert.unimore.it/s/BLPpiAigZJLorQD/download) | +| yolo4x | Yolov4x-mish 9 | [COCO 2017](http://cocodataset.org/) | 80 | 640x640 | [weights](https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download) | +| yolo4x-cps | Scaled Yolov4 10 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/AfzHE4BfTeEm2gH/download) | ## References @@ -367,3 +368,4 @@ This demo also creates a json file named ```net_name_COCO_res.json``` containing 7. Wang, Chien-Yao, et al. "CSPNet: A New Backbone that can Enhance Learning Capability of CNN." arXiv preprint arXiv:1911.11929 (2019). 8. Bochkovskiy, Alexey, Chien-Yao Wang, and Hong-Yuan Mark Liao. "YOLOv4: Optimal Speed and Accuracy of Object Detection." arXiv preprint arXiv:2004.10934 (2020). 9. Bochkovskiy, Alexey, "Yolo v4, v3 and v2 for Windows and Linux" (https://github.com/AlexeyAB/darknet) +10. Wang, Chien-Yao, Alexey Bochkovskiy, and Hong-Yuan Mark Liao. "Scaled-YOLOv4: Scaling Cross Stage Partial Network." arXiv preprint arXiv:2011.08036 (2020). diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 25c4565..e097372 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -19,6 +19,7 @@ enum layerType_t { LAYER_ACTIVATION_CRELU, LAYER_ACTIVATION_LEAKY, LAYER_ACTIVATION_MISH, + LAYER_ACTIVATION_LOGISTIC, LAYER_FLATTEN, LAYER_RESHAPE, LAYER_MULADD, @@ -68,6 +69,7 @@ public: case LAYER_ACTIVATION_CRELU: return "ActivationCReLU"; case LAYER_ACTIVATION_LEAKY: return "ActivationLeaky"; case LAYER_ACTIVATION_MISH: return "ActivationMish"; + case LAYER_ACTIVATION_LOGISTIC: return "ActivationLogistic"; case LAYER_FLATTEN: return "Flatten"; case LAYER_RESHAPE: return "Reshape"; case LAYER_MULADD: return "MulAdd"; @@ -212,7 +214,8 @@ public: typedef enum { ACTIVATION_ELU = 100, ACTIVATION_LEAKY = 101, - ACTIVATION_MISH = 102 + ACTIVATION_MISH = 102, + ACTIVATION_LOGISTIC = 103 } tkdnnActivationMode_t; /** @@ -233,6 +236,8 @@ public: return LAYER_ACTIVATION_LEAKY; else if (act_mode == ACTIVATION_MISH) return LAYER_ACTIVATION_MISH; + else if (act_mode == ACTIVATION_LOGISTIC) + return LAYER_ACTIVATION_LOGISTIC; else return LAYER_ACTIVATION; }; diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index 4c6c816..4fe2e0e 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -24,6 +24,7 @@ template T readBUF(const char*& buffer) using namespace nvinfer1; #include "pluginsRT/ActivationLeakyRT.h" +#include "pluginsRT/ActivationLogisticRT.h" #include "pluginsRT/ActivationReLUCeilingRT.h" #include "pluginsRT/ActivationMishRT.h" #include "pluginsRT/ReorgRT.h" diff --git a/include/tkDNN/pluginsRT/ActivationLogisticRT.h b/include/tkDNN/pluginsRT/ActivationLogisticRT.h new file mode 100644 index 0000000..a1ceb6b --- /dev/null +++ b/include/tkDNN/pluginsRT/ActivationLogisticRT.h @@ -0,0 +1,60 @@ +#include +#include "../kernels.h" + +class ActivationLogisticRT : public IPlugin { + +public: + ActivationLogisticRT() { + + + } + + ~ActivationLogisticRT(){ + + } + + int getNbOutputs() const override { + return 1; + } + + Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override { + return inputs[0]; + } + + void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override { + size = 1; + for(int i=0; i(inputs[0]), + reinterpret_cast(outputs[0]), batchSize*size, stream); + return 0; + } + + + virtual size_t getSerializationSize() override { + return 1*sizeof(int); + } + + virtual void serialize(void* buffer) override { + char *buf = reinterpret_cast(buffer); + tk::dnn::writeBUF(buf, size); + } + + int size; +}; diff --git a/include/tkDNN/pluginsRT/YoloRT.h b/include/tkDNN/pluginsRT/YoloRT.h index 9af8587..f2bdaa1 100644 --- a/include/tkDNN/pluginsRT/YoloRT.h +++ b/include/tkDNN/pluginsRT/YoloRT.h @@ -67,15 +67,17 @@ public: for (int b = 0; b < batchSize; ++b){ for(int n = 0; n < n_masks; ++n){ int index = entry_index(b, n*w*h, 0); - if (new_coords == 1) - activationLOGISTICForward(srcData + index, dstData + index, 4*w*h, stream); //x,y,w,h - else + if (new_coords == 1){ + if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); + } + else{ activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream); //x,y - if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); + if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); - index = entry_index(b, n*w*h, 4); - activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream); + index = entry_index(b, n*w*h, 4); + activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream); + } } } diff --git a/scripts/test_all_tests.sh b/scripts/test_all_tests.sh index af04aff..6aab775 100644 --- a/scripts/test_all_tests.sh +++ b/scripts/test_all_tests.sh @@ -69,10 +69,11 @@ do echo -e "${ORANGE}Batch $TKDNN_BATCHSIZE ${NC}" test_net mnist - ./test_imuodom &>> $out_file - print_output $? imuodom + # ./test_imuodom &>> $out_file + # print_output $? imuodom test_net yolo4 + test_net yolo4-csp test_net yolo4x test_net yolo4_berkeley test_net yolo4tiny diff --git a/src/Activation.cpp b/src/Activation.cpp index 28c7624..a8642e7 100644 --- a/src/Activation.cpp +++ b/src/Activation.cpp @@ -52,6 +52,10 @@ dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) { else if(act_mode == ACTIVATION_MISH) { activationMishForward(srcData, dstData, dim.tot()); + } + else if(act_mode == ACTIVATION_LOGISTIC) { + activationLOGISTICForward(srcData, dstData, dim.tot()); + } else { dnnType alpha = dnnType(1); dnnType beta = dnnType(0); diff --git a/src/DarknetParser.cpp b/src/DarknetParser.cpp index 7b5410c..69b6b29 100644 --- a/src/DarknetParser.cpp +++ b/src/DarknetParser.cpp @@ -187,6 +187,7 @@ namespace tk { namespace dnn { if(f.activation == "relu") act = tkdnnActivationMode_t(CUDNN_ACTIVATION_RELU); else if(f.activation == "leaky") act = tk::dnn::ACTIVATION_LEAKY; else if(f.activation == "mish") act = tk::dnn::ACTIVATION_MISH; + else if(f.activation == "logistic") act = tk::dnn::ACTIVATION_LOGISTIC; else { FatalError("activation not supported: " + f.activation); } netLayers[netLayers.size()-1] = new tk::dnn::Activation(net, act); }; diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 501ade4..c915ba5 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -226,7 +226,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) { return convert_layer(input, (Conv2d*) l); if(type == LAYER_POOLING) return convert_layer(input, (Pooling*) l); - if(type == LAYER_ACTIVATION || type == LAYER_ACTIVATION_CRELU || type == LAYER_ACTIVATION_LEAKY || type == LAYER_ACTIVATION_MISH) + if(type == LAYER_ACTIVATION || type == LAYER_ACTIVATION_CRELU || type == LAYER_ACTIVATION_LEAKY || type == LAYER_ACTIVATION_MISH || type == LAYER_ACTIVATION_LOGISTIC) return convert_layer(input, (Activation*) l); if(type == LAYER_SOFTMAX) return convert_layer(input, (Softmax*) l); @@ -421,6 +421,12 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { checkNULL(lRT); return lRT; } + else if(l->act_mode == ACTIVATION_LOGISTIC) { + IPlugin *plugin = new ActivationLogisticRT(); + IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); + checkNULL(lRT); + return lRT; + } else { FatalError("this Activation mode is not yet implemented"); return NULL; @@ -653,6 +659,11 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa a->size = readBUF(buf); return a; } + if(name.find("ActivationLogistic") == 0) { + ActivationLogisticRT *a = new ActivationLogisticRT(); + a->size = readBUF(buf); + return a; + } if(name.find("ActivationCReLU") == 0) { ActivationReLUCeiling *a = new ActivationReLUCeiling(readBUF(buf)); a->size = readBUF(buf); diff --git a/src/Yolo.cpp b/src/Yolo.cpp index 9737e74..61ed3cf 100644 --- a/src/Yolo.cpp +++ b/src/Yolo.cpp @@ -72,8 +72,8 @@ Yolo::box get_yolo_box(float *x, float *biases, int n, int index, int i, int j, b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h; } else{ - b.x = (i + x[index + 0 * stride] * 2 - 0.5) / lw; - b.y = (j + x[index + 1 * stride] * 2 - 0.5) / lh; + b.x = (i + x[index + 0 * stride] ) / lw; + b.y = (j + x[index + 1 * stride] ) / lh; b.w = x[index + 2 * stride] * x[index + 2 * stride] * 4 * biases[2 * n] / w; b.h = x[index + 3 * stride] * x[index + 3 * stride] * 4 * biases[2 * n + 1] / h; } @@ -87,15 +87,18 @@ dnnType* Yolo::infer(dataDim_t &dim, dnnType* srcData) { for (int b = 0; b < dim.n; ++b){ for(int n = 0; n < n_masks; ++n){ int index = entry_index(b, n*dim.w*dim.h, 0, classes, input_dim, output_dim); - if (new_coords == 1) - activationLOGISTICForward(srcData + index, dstData + index, 4*dim.w*dim.h); - else + std::cout<<"new_coords"<scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); + } + else{ activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h); - if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); - - index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim); - activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h); + if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); + + index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim); + activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h); + } } } diff --git a/tests/darknet/cfg/yolo4-csp.cfg b/tests/darknet/cfg/yolo4-csp.cfg new file mode 100644 index 0000000..691ec03 --- /dev/null +++ b/tests/darknet/cfg/yolo4-csp.cfg @@ -0,0 +1,1279 @@ +[net] +# Testing +#batch=1 +#subdivisions=1 +# Training +batch=64 +subdivisions=8 +width=512 +height=512 +channels=3 +momentum=0.949 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=1000 +max_batches = 500500 +policy=steps +steps=400000,450000 +scales=.1,.1 + +mosaic=1 + +letter_box=1 + +ema_alpha=0.9998 + +#optimized_memory=1 + +#23:104x104 54:52x52 85:26x26 104:13x13 for 416 + + + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=mish + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +#[route] +#layers = -2 + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +#[route] +#layers = -1,-7 + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-10 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-16 + +[convolutional] +batch_normalize=1 +filters=1024 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +### SPP ### +[maxpool] +stride=1 +size=5 + +[route] +layers=-2 + +[maxpool] +stride=1 +size=9 + +[route] +layers=-4 + +[maxpool] +stride=1 +size=13 + +[route] +layers=-1,-3,-5,-6 +### End SPP ### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1, -13 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 79 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1, -6 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 48 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=128 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=128 +activation=mish + +[route] +layers = -1, -6 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=logistic + + +[yolo] +mask = 0,1,2 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=0 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=4.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=5 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1, -20 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1,-6 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=logistic + + +[yolo] +mask = 3,4,5 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=1.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=5 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1, -49 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1,-6 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=logistic + + +[yolo] +mask = 6,7,8 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=0.4 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=2 diff --git a/tests/darknet/cfg/yolo4x.cfg b/tests/darknet/cfg/yolo4x.cfg index 89f2564..2ff854f 100644 --- a/tests/darknet/cfg/yolo4x.cfg +++ b/tests/darknet/cfg/yolo4x.cfg @@ -5,8 +5,8 @@ # Training batch=64 subdivisions=8 -width=672 -height=672 +width=640 +height=640 channels=3 momentum=0.949 decay=0.0005 @@ -15,7 +15,7 @@ saturation = 1.5 exposure = 1.5 hue=.1 -learning_rate=0.00261 +learning_rate=0.001 burn_in=1000 max_batches = 500500 policy=steps @@ -26,6 +26,8 @@ mosaic=1 letter_box=1 +#optimized_memory=1 + [convolutional] batch_normalize=1 filters=32 @@ -1131,6 +1133,7 @@ size=1 stride=1 pad=1 activation=mish +stopbackward=800 ########################## @@ -1147,7 +1150,7 @@ size=1 stride=1 pad=1 filters=255 -activation=linear +activation=logistic [yolo] @@ -1156,6 +1159,7 @@ anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 4 classes=80 num=9 jitter=.1 +scale_x_y = 2.0 objectness_smooth=0 ignore_thresh = .7 truth_thresh = 1 @@ -1169,6 +1173,7 @@ iou_loss=ciou nms_kind=diounms beta_nms=0.6 new_coords=1 +max_delta=5 [route] layers = -4 @@ -1275,7 +1280,7 @@ size=1 stride=1 pad=1 filters=255 -activation=linear +activation=logistic [yolo] @@ -1284,6 +1289,7 @@ anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 4 classes=80 num=9 jitter=.1 +scale_x_y = 2.0 objectness_smooth=1 ignore_thresh = .7 truth_thresh = 1 @@ -1297,6 +1303,7 @@ iou_loss=ciou nms_kind=diounms beta_nms=0.6 new_coords=1 +max_delta=5 [route] layers = -4 @@ -1403,7 +1410,7 @@ size=1 stride=1 pad=1 filters=255 -activation=linear +activation=logistic [yolo] @@ -1412,6 +1419,7 @@ anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 4 classes=80 num=9 jitter=.1 +scale_x_y = 2.0 objectness_smooth=1 ignore_thresh = .7 truth_thresh = 1 @@ -1425,3 +1433,4 @@ iou_loss=ciou nms_kind=diounms beta_nms=0.6 new_coords=1 +max_delta=2 diff --git a/tests/darknet/yolo4-csp.cpp b/tests/darknet/yolo4-csp.cpp new file mode 100644 index 0000000..af8a7fc --- /dev/null +++ b/tests/darknet/yolo4-csp.cpp @@ -0,0 +1,36 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4-csp"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer144_out.bin", + bin_path + "/debug/layer159_out.bin", + bin_path + "/debug/layer174_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4-csp.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/AfzHE4BfTeEm2gH/download"); + + + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} diff --git a/tests/darknet/yolo4x.cpp b/tests/darknet/yolo4x.cpp index b9ad003..8df1aef 100644 --- a/tests/darknet/yolo4x.cpp +++ b/tests/darknet/yolo4x.cpp @@ -17,7 +17,7 @@ int main() { std::string wgs_path = bin_path + "/layers"; std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4x.cfg"; std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/BLPpiAigZJLorQD/download"); + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download"); -- 2.52.0 From fb52444cdc06b6a3868dd13cdd2cc0d0601cc6c0 Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Mon, 25 Jan 2021 04:12:05 +0530 Subject: [PATCH 104/228] Replaced dynamic arrays with std::vector ,works on linux ..needs to be tested on windows after clearing up the lnk2019 error --- CMakeLists.txt | 4 ++-- src/Yolo3Detection.cpp | 7 ++++--- src/utils.cpp | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03e26c5..c11b56f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,13 +2,13 @@ cmake_minimum_required(VERSION 3.5) project (tkDNN) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) -if(LINUX) +if(UNIX) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable") endif() if(WIN32) set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_FLAGS "/O2 /FS ") -endif() +endif(WIN32) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN) # project specific flags diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index b94eea9..0c638e6 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -94,9 +94,10 @@ void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){ void Yolo3Detection::postprocess(const int bi, const bool mAP){ //get yolo outputs - dnnType *rt_out[netRT->pluginFactory->n_yolos]; - for(int i=0; ipluginFactory->n_yolos; i++) - rt_out[i] = (dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi; + std::vector rt_out; + //dnnType *rt_out[netRT->pluginFactory->n_yolos]; + for(int i=0; ipluginFactory->n_yolos; i++) + rt_out.push_back((dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi); float x_ratio = float(originalSize[bi].width) / float(netRT->input_dim.w); float y_ratio = float(originalSize[bi].height) / float(netRT->input_dim.h); diff --git a/src/utils.cpp b/src/utils.cpp index e143cce..bbdf516 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -170,7 +170,6 @@ void getMemUsage(double& vm_usage_kb, double& resident_set_kb){ using std::ios_base; using std::ifstream; using std::string; - SYSTEM_INFO sysInfo; vm_usage_kb = 0.0; resident_set_kb = 0.0; @@ -195,6 +194,7 @@ void getMemUsage(double& vm_usage_kb, double& resident_set_kb){ #ifdef __linux__ long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages #elif _WIN32 +SYSTEM_INFO sysInfo; long page_size_kb = sysInfo.dwPageSize/1024; #endif -- 2.52.0 From 2d4dececb683ffa28f7f8aaf72a2e645e8c45adb Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Tue, 26 Jan 2021 20:17:43 +0400 Subject: [PATCH 105/228] Builds on windows successfully,issues with deserialization and downloading weights --- .gitignore | 3 ++- CMakeLists.txt | 9 +++++---- demo/demo/demo.cpp | 2 +- include/tkDNN/test.h | 3 ++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index b56526f..5be5a73 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,6 @@ build/ *.hdf5 *.pk *.table +cmake-build-release/ demo/COCO_val2017 -demo/BDD100K_val \ No newline at end of file +demo/BDD100K_val diff --git a/CMakeLists.txt b/CMakeLists.txt index c11b56f..4904e5d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,13 +1,14 @@ cmake_minimum_required(VERSION 3.5) -project (tkDNN) +project (tkDNN CUDA) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) if(UNIX) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable") endif() if(WIN32) set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_FLAGS "/O2 /FS ") +set(CMAKE_CXX_FLAGS "/O2 ") +set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) endif(WIN32) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN) @@ -33,7 +34,7 @@ include_directories(${CUDNN_INCLUDE_DIR}) # compile file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu") cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS}) -cuda_add_library(kernels SHARED ${tkdnn_CUSRC}) +add_library(kernels SHARED ${tkdnn_CUSRC}) target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES}) @@ -47,7 +48,7 @@ find_package(OpenCV REQUIRED) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV") # gives problems in cross-compiling, probably malformed cmake config -#find_package(yaml-cpp REQUIRED) +find_package(yaml-cpp REQUIRED) #------------------------------------------------------------------------------- # Build Libraries diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 9f50d0b..609affe 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -1,7 +1,7 @@ #include #include #include /* srand, rand */ -#include +//#include #include #include "CenternetDetection.h" diff --git a/include/tkDNN/test.h b/include/tkDNN/test.h index 13c943e..f1d44bd 100644 --- a/include/tkDNN/test.h +++ b/include/tkDNN/test.h @@ -29,7 +29,8 @@ int testInference(std::vector input_bins, std::vector readBinaryFile(input_bins[0], net->input_dim.tot(), &input_h, &data); // outputs - dnnType *cudnn_out[outputs.size()], *rt_out[outputs.size()]; + //dnnType *cudnn_out[outputs.size()], *rt_out[outputs.size()]; + std::vector cudnn_out,rt_out; tk::dnn::dataDim_t dim1 = net->input_dim; //input dim printCenteredTitle(" CUDNN inference ", '=', 30); { -- 2.52.0 From 4a9031433399b6dbf5d08a6c963f8e3f3a829b72 Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Tue, 26 Jan 2021 23:11:58 +0530 Subject: [PATCH 106/228] minor fixes in test.h --- CMakeLists.txt | 4 ++-- include/tkDNN/test.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4904e5d..c173eca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.5) -project (tkDNN CUDA) +project (tkDNN) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) if(UNIX) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable") @@ -34,7 +34,7 @@ include_directories(${CUDNN_INCLUDE_DIR}) # compile file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu") cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS}) -add_library(kernels SHARED ${tkdnn_CUSRC}) +cuda_add_library(kernels SHARED ${tkdnn_CUSRC}) target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES}) diff --git a/include/tkDNN/test.h b/include/tkDNN/test.h index f1d44bd..e842269 100644 --- a/include/tkDNN/test.h +++ b/include/tkDNN/test.h @@ -40,7 +40,7 @@ int testInference(std::vector input_bins, std::vector TKDNN_TSTOP dim1.print(); } - for(int i=0; idstData; + for(int i=0; idstData); if(netRT != nullptr) { tk::dnn::dataDim_t dim2 = net->input_dim; @@ -51,7 +51,7 @@ int testInference(std::vector input_bins, std::vector TKDNN_TSTOP dim2.print(); } - for(int i=0; ibuffersRT[i+1]; + for(int i=0; ibuffersRT[i+1]); } int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; -- 2.52.0 From f055341af6cf6a3fb914a95a9aa2561fe3d81df7 Mon Sep 17 00:00:00 2001 From: Ricky Medrano Date: Tue, 9 Feb 2021 07:57:57 -0800 Subject: [PATCH 107/228] Minor Readme Changes Added Logistic as a viable activation you can use. Added conf-thresh as the 7th parameter in the ./demo call. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d9927c0..5055e99 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ All models from darknet are now parsed directly from cfg, you still need to expo relu leaky mish + logistic
## Run the demo @@ -217,7 +218,7 @@ Once you have successfully created your rt file, run the demo: ``` In general the demo program takes 7 parameters: ``` -./demo +./demo ``` where * `````` is the rt file generated by a test -- 2.52.0 From 4b3731928c63f802a2804924463c57d33cf244d1 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 25 Feb 2021 09:57:10 +0100 Subject: [PATCH 108/228] Add yolo4_320_coco2 (pedestrian and stop sign) Signed-off-by: Micaela Verucchi --- tests/darknet/cfg/yolo4_320_coco2.cfg | 1158 +++++++++++++++++++++++++ tests/darknet/names/coco2.names | 2 + tests/darknet/yolo4_320_coco2.cpp | 34 + 3 files changed, 1194 insertions(+) create mode 100644 tests/darknet/cfg/yolo4_320_coco2.cfg create mode 100644 tests/darknet/names/coco2.names create mode 100644 tests/darknet/yolo4_320_coco2.cpp diff --git a/tests/darknet/cfg/yolo4_320_coco2.cfg b/tests/darknet/cfg/yolo4_320_coco2.cfg new file mode 100644 index 0000000..9585fca --- /dev/null +++ b/tests/darknet/cfg/yolo4_320_coco2.cfg @@ -0,0 +1,1158 @@ +[net] +batch=64 +subdivisions=32 +# Training +#width=512 +#height=512 +width=320 +height=320 +channels=3 +momentum=0.949 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.0013 +burn_in=1000 +max_batches = 6000 +policy=steps +steps=4800,5400 +scales=.1,.1 + +#cutmix=1 +mosaic=1 + +#:104x104 54:52x52 85:26x26 104:13x13 for 416 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-7 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-10 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-16 + +[convolutional] +batch_normalize=1 +filters=1024 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +### SPP ### +[maxpool] +stride=1 +size=5 + +[route] +layers=-2 + +[maxpool] +stride=1 +size=9 + +[route] +layers=-4 + +[maxpool] +stride=1 +size=13 + +[route] +layers=-1,-3,-5,-6 +### End SPP ### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = 85 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = 54 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +########################## + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=21 +activation=linear + + +[yolo] +mask = 0,1,2 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=2 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +scale_x_y = 1.2 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=256 +activation=leaky + +[route] +layers = -1, -16 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=21 +activation=linear + + +[yolo] +mask = 3,4,5 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=2 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +scale_x_y = 1.1 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=512 +activation=leaky + +[route] +layers = -1, -37 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=21 +activation=linear + + +[yolo] +mask = 6,7,8 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=2 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 +scale_x_y = 1.05 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + diff --git a/tests/darknet/names/coco2.names b/tests/darknet/names/coco2.names new file mode 100644 index 0000000..e2f8903 --- /dev/null +++ b/tests/darknet/names/coco2.names @@ -0,0 +1,2 @@ +person +stop sign \ No newline at end of file diff --git a/tests/darknet/yolo4_320_coco2.cpp b/tests/darknet/yolo4_320_coco2.cpp new file mode 100644 index 0000000..877e604 --- /dev/null +++ b/tests/darknet/yolo4_320_coco2.cpp @@ -0,0 +1,34 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4_320_coco2"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer139_out.bin", + bin_path + "/debug/layer150_out.bin", + bin_path + "/debug/layer161_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = "../tests/darknet/cfg/yolo4_320_coco2.cfg"; + std::string name_path = "../tests/darknet/names/coco2.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/f3wk99iG5y7tEr8/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} -- 2.52.0 From 6aa8666be54ce7726654afd3062704555a836161 Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Wed, 10 Mar 2021 12:59:04 +0530 Subject: [PATCH 109/228] Commits for msvc 16.9 --- CMakeLists.txt | 2 +- demo/demo/demo.cpp | 2 +- src/kernels/deformable_conv.cu | 2 +- src/utils.cpp | 11 ++++++++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c173eca..77425d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declara endif() if(WIN32) set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_FLAGS "/O2 ") +set(CMAKE_CXX_FLAGS "/O2 /FS ") set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) endif(WIN32) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 609affe..c622f2b 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -22,7 +22,7 @@ int main(int argc, char *argv[]) { signal(SIGINT, sig_handler); - std::string net = "yolo3_berkeley.rt"; + std::string net = "yolo4tiny_fp32.rt"; if(argc > 1) net = argv[1]; std::string input = "../demo/yolo_test.mp4"; diff --git a/src/kernels/deformable_conv.cu b/src/kernels/deformable_conv.cu index 592c538..4dbc552 100644 --- a/src/kernels/deformable_conv.cu +++ b/src/kernels/deformable_conv.cu @@ -18,7 +18,7 @@ inline int GET_BLOCKS(const int N) } -__device__ float dmcn_im2col_bilinear(const float *bottom_data, const int data_width, +__device__ __host__ float dmcn_im2col_bilinear(const float *bottom_data, const int data_width, const int height, const int width, float h, float w) { int h_low = floor(h); int w_low = floor(w); diff --git a/src/utils.cpp b/src/utils.cpp index bbdf516..52775df 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -23,14 +23,23 @@ bool fileExist(const char *fname) { void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string& test_folder, const std::string& weights_url){ if(!fileExist(input_bin.c_str())){ std::string mkdir_cmd = "mkdir " + test_folder; - std::string wget_cmd = "wget " + weights_url + " -O " + test_folder + "/weights.zip"; + std::string wget_cmd = "curl " + weights_url + " --output " + test_folder + "/weights.zip"; +#ifdef __linux__ std::string unzip_cmd = "unzip " + test_folder + "/weights.zip -d" + test_folder; std::string rm_cmd = "rm " + test_folder + "/weights.zip"; + +#elif _WIN32 + + std::string unzip_cmd = "7z x " + test_folder + "/weights.zip -o" + test_folder; +#endif int err = 0; err = system(mkdir_cmd.c_str()); err = system(wget_cmd.c_str()); err = system(unzip_cmd.c_str()); +#ifdef __linux__ err = system(rm_cmd.c_str()); +#endif + } } -- 2.52.0 From 304ab49897938beb3ea6619fb5720bc1aae1280d Mon Sep 17 00:00:00 2001 From: Harshvardhan Chandirasekar Date: Wed, 17 Mar 2021 22:26:26 +0530 Subject: [PATCH 110/228] shared_ptr migrations --- CMakeLists.txt | 2 +- include/tkDNN/NetworkRT.h | 7 ++++++- include/tkDNN/utils.h | 11 +++++++++++ src/NetworkRT.cpp | 11 +++++++---- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 77425d3..f478cab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.5) project (tkDNN) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) if(UNIX) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -fPIC -Wno-deprecated-declarations -Wno-unused-variable") endif() if(WIN32) set(CMAKE_CXX_STANDARD 14) diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index 4c6c816..ca012fb 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -6,6 +6,7 @@ #include "Network.h" #include "Layer.h" #include "NvInfer.h" +#include namespace tk { namespace dnn { @@ -59,7 +60,8 @@ public: #if NV_TENSORRT_MAJOR >= 6 nvinfer1::IBuilderConfig *configRT; #endif - nvinfer1::ICudaEngine *engineRT; + std::shared_ptr engineRT; + //nvinfer1::ICudaEngine *engineRT; nvinfer1::IExecutionContext *contextRT; const static int MAX_BUFFERS_RT = 10; @@ -114,6 +116,9 @@ public: bool serialize(const char *filename); bool deserialize(const char *filename); + + + }; }} diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index 1404509..edc770e 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -110,6 +110,17 @@ double t_ns = time_ms.count(); FatalError(_error.str()); \ } \ } +struct InferDeleter +{ + template + void operator()(T* obj) const + { + if (obj) + { + obj->destroy(); + } + } +}; typedef enum { ERROR_CUDNN = 2, diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 501ade4..4006caf 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -137,9 +137,11 @@ NetworkRT::NetworkRT(Network *net, const char *name) { printCudaMemUsage(); std::cout<<"Building tensorRT cuda engine...\n"; #if NV_TENSORRT_MAJOR >= 6 - engineRT = builderRT->buildEngineWithConfig(*networkRT, *configRT); + //engineRT = builderRT->buildEngineWithConfig(*networkRT, *configRT); + engineRT = std::shared_ptr(builderRT->buildEngineWithConfig(*networkRT,*configRT),InferDeleter()); #else - engineRT = builderRT->buildCudaEngine(*networkRT); + //engineRT = builderRT->buildCudaEngine(*networkRT); + engineRT = std::shared_ptr(builderRT->buildCudaEngine(*networkRT)); #endif if(engineRT == nullptr) FatalError("cloud not build cuda engine") @@ -561,7 +563,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, DeformConv2d *l) { IPluginLayer *lRT = networkRT->addPlugin(inputs, 2, *plugin); checkNULL(lRT); lRT->setName( ("Deformable" + std::to_string(l->id)).c_str() ); - delete(inputs); + delete[](inputs); // batchnorm void *bias_b, *power_b, *mean_b, *variance_b, *scales_b; if(dtRT == DataType::kHALF) { @@ -629,7 +631,8 @@ bool NetworkRT::deserialize(const char *filename) { pluginFactory = new PluginFactory(); runtimeRT = createInferRuntime(loggerRT); - engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size, (IPluginFactory *) pluginFactory); + //engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size, (IPluginFactory *) pluginFactory); + engineRT = std::shared_ptr(runtimeRT->deserializeCudaEngine(gieModelStream,size,(IPluginFactory*)pluginFactory),InferDeleter()); //if (gieModelStream) delete [] gieModelStream; return true; -- 2.52.0 From 06787a931fd8897ce8400d62c60393b8ef5fdc85 Mon Sep 17 00:00:00 2001 From: Harshvardhan Chandirasekar Date: Wed, 17 Mar 2021 23:06:50 +0530 Subject: [PATCH 111/228] minor migrations --- include/tkDNN/NetworkRT.h | 6 ++++-- src/NetworkRT.cpp | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index ca012fb..a50cd8c 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -54,8 +54,10 @@ class NetworkRT { public: nvinfer1::DataType dtRT; - nvinfer1::IBuilder *builderRT; - nvinfer1::IRuntime *runtimeRT; + //nvinfer1::IBuilder *builderRT; + std::unique_ptr builderRT; + //nvinfer1::IRuntime *runtimeRT; + std::unique_ptr runtimeRT; nvinfer1::INetworkDefinition *networkRT; #if NV_TENSORRT_MAJOR >= 6 nvinfer1::IBuilderConfig *configRT; diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 4006caf..e6eb778 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -33,7 +33,8 @@ NetworkRT::NetworkRT(Network *net, const char *name) { float(NV_TENSORRT_PATCH)/100; std::cout<<"New NetworkRT (TensorRT v"<(createInferBuilder(loggerRT)); + //builderRT = createInferBuilder(loggerRT); std::cout<<"Float16 support: "<platformHasFastFp16()<<"\n"; std::cout<<"Int8 support: "<platformHasFastInt8()<<"\n"; #if NV_TENSORRT_MAJOR >= 5 @@ -630,7 +631,8 @@ bool NetworkRT::deserialize(const char *filename) { } pluginFactory = new PluginFactory(); - runtimeRT = createInferRuntime(loggerRT); + //runtimeRT = createInferRuntime(loggerRT); + runtimeRT = std::unique_ptr(createInferRuntime(loggerRT)); //engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size, (IPluginFactory *) pluginFactory); engineRT = std::shared_ptr(runtimeRT->deserializeCudaEngine(gieModelStream,size,(IPluginFactory*)pluginFactory),InferDeleter()); //if (gieModelStream) delete [] gieModelStream; -- 2.52.0 From e94e1f7622d5bc4479036b39e012213acf9c1fdb Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Wed, 24 Mar 2021 22:26:46 +0530 Subject: [PATCH 112/228] tkdnn first patch for windows --- .gitignore | 1 + demo/demo/demo.cpp | 2 +- include/tkDNN/DetectionNN.h | 1 - include/tkDNN/ImuOdom.h | 2 + include/tkDNN/pluginsRT/ActivationLeakyRT.h | 3 +- include/tkDNN/pluginsRT/ActivationMishRT.h | 3 +- .../tkDNN/pluginsRT/ActivationReLUCeilingRT.h | 3 +- include/tkDNN/pluginsRT/ActivationSigmoidRT.h | 3 +- include/tkDNN/pluginsRT/DeformableConvRT.h | 3 +- include/tkDNN/pluginsRT/FlattenConcatRT.h | 3 +- .../tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h | 3 +- include/tkDNN/pluginsRT/RegionRT.h | 3 +- include/tkDNN/pluginsRT/ReorgRT.h | 3 +- include/tkDNN/pluginsRT/ReshapeRT.h | 3 +- include/tkDNN/pluginsRT/ResizeLayerRT.h | 3 +- include/tkDNN/pluginsRT/RouteRT.h | 3 +- include/tkDNN/pluginsRT/ShortcutRT.h | 3 +- include/tkDNN/pluginsRT/UpsampleRT.h | 5 +- include/tkDNN/pluginsRT/YoloRT.h | 36 +++--- src/NetworkRT.cpp | 119 ++++++++++++++---- 20 files changed, 151 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index 5be5a73..7e5c4ca 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ build/ cmake-build-release/ demo/COCO_val2017 demo/BDD100K_val +/.vs diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index c622f2b..f97ead9 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -131,7 +131,7 @@ int main(int argc, char *argv[]) { double mean = 0; std::cout<stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; + std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); std::cout<<"Avg: "< #elif _WIN32 +#define _USE_MATH_DEFINES +#include #include #endif diff --git a/include/tkDNN/pluginsRT/ActivationLeakyRT.h b/include/tkDNN/pluginsRT/ActivationLeakyRT.h index d3f66fb..9e26b2b 100644 --- a/include/tkDNN/pluginsRT/ActivationLeakyRT.h +++ b/include/tkDNN/pluginsRT/ActivationLeakyRT.h @@ -52,8 +52,9 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, size); + assert(buf == a + getSerializationSize()); } int size; diff --git a/include/tkDNN/pluginsRT/ActivationMishRT.h b/include/tkDNN/pluginsRT/ActivationMishRT.h index 1744ab0..5d660af 100644 --- a/include/tkDNN/pluginsRT/ActivationMishRT.h +++ b/include/tkDNN/pluginsRT/ActivationMishRT.h @@ -52,8 +52,9 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, size); + assert(buf == a + getSerializationSize()); } int size; diff --git a/include/tkDNN/pluginsRT/ActivationReLUCeilingRT.h b/include/tkDNN/pluginsRT/ActivationReLUCeilingRT.h index 286f22e..50ceb81 100644 --- a/include/tkDNN/pluginsRT/ActivationReLUCeilingRT.h +++ b/include/tkDNN/pluginsRT/ActivationReLUCeilingRT.h @@ -51,9 +51,10 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, ceiling); tk::dnn::writeBUF(buf, size); + assert(buf = a + getSerializationSize()); } diff --git a/include/tkDNN/pluginsRT/ActivationSigmoidRT.h b/include/tkDNN/pluginsRT/ActivationSigmoidRT.h index 1d47136..bcc58c7 100644 --- a/include/tkDNN/pluginsRT/ActivationSigmoidRT.h +++ b/include/tkDNN/pluginsRT/ActivationSigmoidRT.h @@ -52,8 +52,9 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, size); + assert(buf == a + getSerializationSize()); } int size; diff --git a/include/tkDNN/pluginsRT/DeformableConvRT.h b/include/tkDNN/pluginsRT/DeformableConvRT.h index 225a24e..5cb2bab 100644 --- a/include/tkDNN/pluginsRT/DeformableConvRT.h +++ b/include/tkDNN/pluginsRT/DeformableConvRT.h @@ -116,7 +116,7 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, chunk_dim); tk::dnn::writeBUF(buf, kh); tk::dnn::writeBUF(buf, kw); @@ -163,6 +163,7 @@ public: for(int i=0; i(buffer); + char *buf = reinterpret_cast(buffer),*a = buf; tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); tk::dnn::writeBUF(buf, rows); tk::dnn::writeBUF(buf, cols); + assert(buf == a + getSerializationSize()); } int c, h, w; diff --git a/include/tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h b/include/tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h index 911fca2..0899a34 100644 --- a/include/tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h +++ b/include/tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h @@ -55,7 +55,7 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, this->c); tk::dnn::writeBUF(buf, this->h); @@ -65,6 +65,7 @@ public: tk::dnn::writeBUF(buf, this->stride_W); tk::dnn::writeBUF(buf, this->winSize); tk::dnn::writeBUF(buf, this->padding); + assert(buf == a + getSerializationSize()); } int n, c, h, w; diff --git a/include/tkDNN/pluginsRT/RegionRT.h b/include/tkDNN/pluginsRT/RegionRT.h index f0d127e..8487652 100644 --- a/include/tkDNN/pluginsRT/RegionRT.h +++ b/include/tkDNN/pluginsRT/RegionRT.h @@ -73,13 +73,14 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, classes); tk::dnn::writeBUF(buf, coords); tk::dnn::writeBUF(buf, num); tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); + assert(buf == a + getSerializationSize()); } int c, h, w; diff --git a/include/tkDNN/pluginsRT/ReorgRT.h b/include/tkDNN/pluginsRT/ReorgRT.h index ee85718..c1b529a 100644 --- a/include/tkDNN/pluginsRT/ReorgRT.h +++ b/include/tkDNN/pluginsRT/ReorgRT.h @@ -52,11 +52,12 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, stride); tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); + assert(buf == a + getSerializationSize()); } int c, h, w, stride; diff --git a/include/tkDNN/pluginsRT/ReshapeRT.h b/include/tkDNN/pluginsRT/ReshapeRT.h index 97030db..37017c7 100644 --- a/include/tkDNN/pluginsRT/ReshapeRT.h +++ b/include/tkDNN/pluginsRT/ReshapeRT.h @@ -50,11 +50,12 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a = buf; tk::dnn::writeBUF(buf, n); tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); + assert(buf == a + getSerializationSize()); } int n, c, h, w; diff --git a/include/tkDNN/pluginsRT/ResizeLayerRT.h b/include/tkDNN/pluginsRT/ResizeLayerRT.h index ae87dbf..cde52bf 100644 --- a/include/tkDNN/pluginsRT/ResizeLayerRT.h +++ b/include/tkDNN/pluginsRT/ResizeLayerRT.h @@ -52,7 +52,7 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, o_c); tk::dnn::writeBUF(buf, o_h); @@ -61,6 +61,7 @@ public: tk::dnn::writeBUF(buf, i_c); tk::dnn::writeBUF(buf, i_h); tk::dnn::writeBUF(buf, i_w); + assert(buf == a + getSerializationSize()); } int i_c, i_h, i_w, o_c, o_h, o_w; diff --git a/include/tkDNN/pluginsRT/RouteRT.h b/include/tkDNN/pluginsRT/RouteRT.h index 23f30b7..5a8c170 100644 --- a/include/tkDNN/pluginsRT/RouteRT.h +++ b/include/tkDNN/pluginsRT/RouteRT.h @@ -75,7 +75,7 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, groups); tk::dnn::writeBUF(buf, group_id); tk::dnn::writeBUF(buf, in); @@ -85,6 +85,7 @@ public: tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); + assert(buf == a + getSerializationSize()); } static const int MAX_INPUTS = 4; diff --git a/include/tkDNN/pluginsRT/ShortcutRT.h b/include/tkDNN/pluginsRT/ShortcutRT.h index 3eadd3f..17f050f 100644 --- a/include/tkDNN/pluginsRT/ShortcutRT.h +++ b/include/tkDNN/pluginsRT/ShortcutRT.h @@ -59,13 +59,14 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, bc); tk::dnn::writeBUF(buf, bh); tk::dnn::writeBUF(buf, bw); tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); + assert(buf == a + getSerializationSize()); } diff --git a/include/tkDNN/pluginsRT/UpsampleRT.h b/include/tkDNN/pluginsRT/UpsampleRT.h index 7a62abc..5350b7e 100644 --- a/include/tkDNN/pluginsRT/UpsampleRT.h +++ b/include/tkDNN/pluginsRT/UpsampleRT.h @@ -54,11 +54,14 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); + char *buf = reinterpret_cast(buffer),*a=buf; tk::dnn::writeBUF(buf, stride); tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); + std::cout << "Upsample Serialization SIze" << getSerializationSize() << std::endl; + + assert(buf == a + getSerializationSize()); } int c, h, w, stride; diff --git a/include/tkDNN/pluginsRT/YoloRT.h b/include/tkDNN/pluginsRT/YoloRT.h index 9af8587..451d99f 100644 --- a/include/tkDNN/pluginsRT/YoloRT.h +++ b/include/tkDNN/pluginsRT/YoloRT.h @@ -89,21 +89,25 @@ public: } virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); - tk::dnn::writeBUF(buf, classes); - tk::dnn::writeBUF(buf, num); - tk::dnn::writeBUF(buf, n_masks); - tk::dnn::writeBUF(buf, scaleXY); - tk::dnn::writeBUF(buf, nms_thresh); - tk::dnn::writeBUF(buf, nms_kind); - tk::dnn::writeBUF(buf, new_coords); - tk::dnn::writeBUF(buf, c); - tk::dnn::writeBUF(buf, h); - tk::dnn::writeBUF(buf, w); - for(int i=0; i(buffer),*a=buf; + tk::dnn::writeBUF(buf, classes); std::cout << "Classes :" << classes << std::endl; + tk::dnn::writeBUF(buf, num); std::cout << "Num : " << num << std::endl; + tk::dnn::writeBUF(buf, n_masks); std::cout << "N_Masks" << n_masks << std::endl; + tk::dnn::writeBUF(buf, scaleXY); std::cout << "ScaleXY :" << scaleXY << std::endl; + tk::dnn::writeBUF(buf, nms_thresh); std::cout << "nms_thresh :" << nms_thresh << std::endl; + tk::dnn::writeBUF(buf, nms_kind); std::cout << "nms_kind : " << nms_kind << std::endl; + tk::dnn::writeBUF(buf, new_coords); std::cout << "new_coords : " << new_coords << std::endl; + tk::dnn::writeBUF(buf, c); std::cout << "C : " << c << std::endl; + tk::dnn::writeBUF(buf, h); std::cout << "H : " << h << std::endl; + tk::dnn::writeBUF(buf, w); std::cout << "C : " << c << std::endl; + for (int i = 0; i < n_masks; i++) + { + tk::dnn::writeBUF(buf, mask[i]); std::cout << "mask[i] : " << mask[i] << std::endl; + } + for (int i = 0; i < n_masks * 2 * num; i++) + { + tk::dnn::writeBUF(buf, bias[i]); std::cout << "bias[i] : " << bias[i] << std::endl; + } // save classes names for(int i=0; i(serialData); + const char * buf = reinterpret_cast(serialData),*bufCheck = buf; std::string name(layerName); - //std::cout<size = readBUF(buf); + assert(buf == bufCheck + serialLength); return a; } if(name.find("ActivationMish") == 0) { ActivationMishRT *a = new ActivationMishRT(); a->size = readBUF(buf); + assert(buf == bufCheck + serialLength); return a; } if(name.find("ActivationCReLU") == 0) { - ActivationReLUCeiling *a = new ActivationReLUCeiling(readBUF(buf)); + float activationReluTemp = readBUF(buf); + //ActivationReLUCeiling *a = new ActivationReLUCeiling(readBUF(buf)); + ActivationReLUCeiling* a = new ActivationReLUCeiling(activationReluTemp); a->size = readBUF(buf); + assert(buf == bufCheck + serialLength); return a; } if(name.find("Region") == 0) { - RegionRT *r = new RegionRT(readBUF(buf), //classes + int classesTemp = readBUF(buf); + int coordsTemp = readBUF(buf); + int numTemp = readBUF(buf); + /*RegionRT *r = new RegionRT(readBUF(buf), //classes readBUF(buf), //coords - readBUF(buf)); //num + readBUF(buf)); //num8*/ + RegionRT* r = new RegionRT(classesTemp, coordsTemp, numTemp); r->c = readBUF(buf); r->h = readBUF(buf); r->w = readBUF(buf); + assert(buf == bufCheck + serialLength); return r; } if(name.find("Reorg") == 0) { - ReorgRT *r = new ReorgRT(readBUF(buf)); //stride + int strideTemp = readBUF(buf); + //ReorgRT *r = new ReorgRT(readBUF(buf)); //stride + ReorgRT *r = new ReorgRT(strideTemp); r->c = readBUF(buf); r->h = readBUF(buf); r->w = readBUF(buf); + assert(buf == bufCheck + serialLength); return r; } @@ -695,27 +708,46 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa r->h = readBUF(buf); r->w = readBUF(buf); return r; + assert(buf == bufCheck + serialLength); } if(name.find("Pooling") == 0) { - MaxPoolFixedSizeRT *r = new MaxPoolFixedSizeRT( readBUF(buf), //c + /* MaxPoolFixedSizeRT *r = new MaxPoolFixedSizeRT( readBUF(buf), //c readBUF(buf), //h readBUF(buf), //w readBUF(buf), //n readBUF(buf), //strideH readBUF(buf), //strideW readBUF(buf), //winSize - readBUF(buf)); //padding + readBUF(buf)); //padding*/ + + int cTemp = readBUF(buf); + int hTemp = readBUF(buf); + int wTemp = readBUF(buf); + int nTemp = readBUF(buf); + int strideHTemp = readBUF(buf); + int strideWTemp = readBUF(buf); + int winSizeTemp = readBUF(buf); + int paddingTemp = readBUF(buf); + + MaxPoolFixedSizeRT* r = new MaxPoolFixedSizeRT(cTemp, hTemp, wTemp, nTemp, strideHTemp, strideWTemp, winSizeTemp, paddingTemp); + assert(buf == bufCheck + serialLength); return r; } if(name.find("Resize") == 0) { - ResizeLayerRT *r = new ResizeLayerRT(readBUF(buf), //o_c + /*ResizeLayerRT *r = new ResizeLayerRT(readBUF(buf), //o_c readBUF(buf), //o_h - readBUF(buf)); //o_w + readBUF(buf)); //o_w*/ + int o_cTemp = readBUF(buf); + int o_hTemp = readBUF(buf); + int o_wTemp = readBUF(buf); + ResizeLayerRT* r = new ResizeLayerRT(o_cTemp, o_hTemp, o_wTemp); + r->i_c = readBUF(buf); r->i_h = readBUF(buf); r->i_w = readBUF(buf); + assert(buf == bufCheck + serialLength); return r; } @@ -726,6 +758,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa r->w = readBUF(buf); r->rows = readBUF(buf); r->cols = readBUF(buf); + assert(buf == bufCheck + serialLength); return r; } @@ -737,20 +770,33 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa new_dim.h = readBUF(buf); new_dim.w = readBUF(buf); ReshapeRT *r = new ReshapeRT(new_dim); + assert(buf == bufCheck + serialLength); return r; } if(name.find("Yolo") == 0) { - YoloRT *r = new YoloRT(readBUF(buf), //classes - readBUF(buf), //num - nullptr, //yolo - readBUF(buf), //n_masks - readBUF(buf), //scale_xy - readBUF(buf), //nms_thresh - readBUF(buf), //nms_kind - readBUF(buf) //new_coords - ); + + int classes_temp = readBUF(buf); + int num_temp = readBUF(buf); + int n_masks_temp = readBUF(buf); + float scale_xy_temp = readBUF(buf); + float nms_thresh_temp = readBUF(buf); + int nms_kind_temp = readBUF(buf); + int new_coords_temp = readBUF(buf); + std::cout << classes_temp << ":" << num_temp << ":" << ":" << n_masks_temp << ":" << nms_thresh_temp << ":" << nms_kind_temp << ":" << new_coords_temp << std::endl; + + YoloRT *r = new YoloRT(classes_temp,num_temp,nullptr,n_masks_temp,scale_xy_temp,nms_thresh_temp,nms_kind_temp,new_coords_temp); + + /* std::cout << "classes : " << r->classes; + std::cout << "num : " << r->num; + std::cout << "n_masks : " << r->n_masks; + std::cout << "scalexy : " << r->scaleXY; + std::cout << "nms_thresh : " << r->nms_thresh; + std::cout << "nms_kind : " << r->nms_kind; + std::cout << "new_coords : " << r->new_coords;*/ + + r->c = readBUF(buf); r->h = readBUF(buf); r->w = readBUF(buf); @@ -767,36 +813,62 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa tmp[j] = readBUF(buf); r->classesNames[i] = std::string(tmp); } + assert(buf == bufCheck + serialLength); yolos[n_yolos++] = r; return r; } if(name.find("Upsample") == 0) { - UpsampleRT *r = new UpsampleRT(readBUF(buf)); //stride + //UpsampleRT *r = new UpsampleRT(readBUF(buf)); //stride + int strideTemp = readBUF(buf); + UpsampleRT* r = new UpsampleRT(strideTemp); r->c = readBUF(buf); r->h = readBUF(buf); r->w = readBUF(buf); + assert(buf == bufCheck + serialLength); return r; } if(name.find("Route") == 0) { - RouteRT *r = new RouteRT(readBUF(buf),readBUF(buf)); + //RouteRT *r = new RouteRT(readBUF(buf),readBUF(buf)); + int groupsTemp = readBUF(buf); + int group_idTemp = readBUF(buf); + RouteRT* r = new RouteRT(groupsTemp, group_idTemp); r->in = readBUF(buf); for(int i=0; ic_in[i] = readBUF(buf); r->c = readBUF(buf); r->h = readBUF(buf); r->w = readBUF(buf); + assert(buf == bufCheck + serialLength); return r; } if(name.find("Deformable") == 0) { - DeformableConvRT *r = new DeformableConvRT(readBUF(buf), readBUF(buf), readBUF(buf), + /*DeformableConvRT *r = new DeformableConvRT(readBUF(buf), readBUF(buf), readBUF(buf), readBUF(buf), readBUF(buf), readBUF(buf), readBUF(buf), readBUF(buf), readBUF(buf),readBUF(buf),readBUF(buf),readBUF(buf), readBUF(buf),readBUF(buf),readBUF(buf),readBUF(buf), - nullptr); + nullptr); */ + int chuck_dimTemp = readBUF(buf); + int khTemp = readBUF(buf); + int kwTemp = readBUF(buf); + int shTemp = readBUF(buf); + int swTemp = readBUF(buf); + int phTemp = readBUF(buf); + int pwTemp = readBUF(buf); + int deformableGroupTemp = readBUF(buf); + int i_nTemp = readBUF(buf); + int i_cTemp = readBUF(buf); + int i_hTemp = readBUF(buf); + int i_wTemp = readBUF(buf); + int o_nTemp = readBUF(buf); + int o_cTemp = readBUF(buf); + int o_hTemp = readBUF(buf); + int o_wTemp = readBUF(buf); + + DeformableConvRT* r = new DeformableConvRT(chuck_dimTemp, khTemp, kwTemp, shTemp, swTemp, phTemp, pwTemp, deformableGroupTemp, i_nTemp, i_cTemp, i_hTemp, i_wTemp, o_nTemp, o_cTemp, o_hTemp, o_wTemp, nullptr); dnnType *aus = new dnnType[r->chunk_dim*2]; for(int i=0; ichunk_dim*2; i++) aus[i] = readBUF(buf); @@ -827,6 +899,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa aus[i] = readBUF(buf); checkCuda( cudaMemcpy(r->ones_d2, aus, sizeof(dnnType)*r->dim_ones, cudaMemcpyHostToDevice) ); free(aus); + assert(buf == bufCheck + serialLength); return r; } -- 2.52.0 From 78859fe19109f265489090911ee57629bf31ff67 Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Thu, 25 Mar 2021 16:38:53 +0530 Subject: [PATCH 113/228] timer fix --- .gitignore | 4 ++++ CMakeLists.txt | 2 +- include/tkDNN/utils.h | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7e5c4ca..c1d362c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,7 @@ cmake-build-release/ demo/COCO_val2017 demo/BDD100K_val /.vs +cmake-build-minsizerel/* +scripts/COCO_val2017/* +scripts/COCO_val2017.zip +scripts/all_labels.txt \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index f478cab..579a8bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -fPIC -Wno-deprecated-declara endif() if(WIN32) set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_FLAGS "/O2 /FS ") +set(CMAKE_CXX_FLAGS "/O1 /FS") set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) endif(WIN32) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN) diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index edc770e..ce4a617 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -60,7 +60,7 @@ #define TKDNN_TSTART auto start = std::chrono::high_resolution_clock::now(); #define TKDNN_TSTOP auto stop = std::chrono::high_resolution_clock::now(); \ std::chrono::duration duration = stop -start; \ -auto time_ms = std::chrono::duration_cast(duration);\ +auto time_ms = std::chrono::duration_cast(duration);\ double t_ns = time_ms.count(); #endif -- 2.52.0 From 44b71ae6f33acbaf09fd8a4064d15b8395768cfb Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Thu, 25 Mar 2021 20:44:43 +0530 Subject: [PATCH 114/228] ReadMe.md windows changes --- CMakeLists.txt | 2 +- README.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 579a8bc..1b7ed63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -fPIC -Wno-deprecated-declara endif() if(WIN32) set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_FLAGS "/O1 /FS") +set(CMAKE_CXX_FLAGS "/O1 /FS /EHsc") set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) endif(WIN32) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN) diff --git a/README.md b/README.md index 84e0037..242220a 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,13 @@ Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001 - [mAP demo](#map-demo) - [Existing tests and supported networks](#existing-tests-and-supported-networks) - [References](#references) + - [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows) + - [Dependencies](#dependencies) + - [Compiling tkDNN on Windows](#tkdnn-windows-compile) + - [Run the demo on Windows](#run-the-demo-on-windows) + - [FP16 interference windows](#fp16-windows) + - [INT8 interference windows](#int8-windows) + @@ -355,6 +362,84 @@ This demo also creates a json file named ```net_name_COCO_res.json``` containing | yolo4tiny | Yolov4 tiny 9 | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | | yolo4x | Yolov4x-mish 9 | [COCO 2017](http://cocodataset.org/) | 80 | 672x672 | [weights](https://cloud.hipert.unimore.it/s/BLPpiAigZJLorQD/download) | +##tkDNN on Windows 10 (experimental) + +### Dependencies +This branch should work on every NVIDIA GPU supported in windows with the following dependencies: + +* WINDOWS 10 1803 or HIGHER +* CUDA 10.0 (Recommended CUDA 11.0 +) +* CUDNN 7.6 (Recommended CUDNN 8.0.0 +) +* TENSORRT 6.0.1 (Recommended TENSORRT 7.1 +) +* OPENCV 3.4 (Recommended OPENCV 4.2.0 +) +* MSVC 16.7 (Recommended MSVC 16.8/16.9) +* YAML-CPP 0.5.2 +* EIGEN3 +* 7ZIP (ADD TO PATH) +* NINJA 1.10 + +All the above mentioned dependencies except 7ZIP can be installed using Microsoft's [VCPKG](https://github.com/microsoft/vcpkg.git) . +After bootstrapping VCPKG the dependencies can be built and installed using the following command : + +```vcpkg.exe install opencv4[tbb,jpeg,tiff,opengl,openmp,png,ffmpeg]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build``` + +After VCPKG finishes building and installing all the packages delete C:\temp_vcpkg_build and add C:\opt\x64-windows\bin and C:\opt\x64-windows\debug\bin to path + +### Compiling tkDNN on Windows + +tkDNN is built with cmake(3.15+) on windows along with ninja.Msbuild and NMake Makefiles are drastically slower when compiling the library compared to windows +``` +git clone https://git.hipert.unimore.it/research-cv-chandirasekar/tkdnn-windows.git +cd tkdnn-windows +mkdir build +cd build +cmake -DCMAKE_BUILD_TYPE=Release -G"Ninja" .. +ninja -j4 +``` + +### Run the demo on Windows + +This example uses yolo4_tiny.\ +To run the object detection file create .rt file bu running: +``` +.\test_yolo4tiny.exe +``` + +Once the rt file has been successfully create,run the demo using the following command: +``` +.\demo.exe yolo4tiny_fp32.rt ..\demo\yolo_test.mp4 y +``` + For general info on more demo paramters,check Run the demo section on top + +### FP16 interference windows + +This is an untested feature on windows.To run the object detection demo with FP16 interference follow the below steps(example with yolo4tiny): +``` +set TKDNN_MODE=FP16 +del /f yolo4tiny_fp16.rt +.\test_yolo4tiny.exe +.\demo.exe yolo4tiny_fp16.rt ..\demo\yolo_test.mp4 +``` + +### INT8 interference windows +To run object detection demo with INT8 (example with yolo4tiny): +``` +set TKDNN_MODE=INT8 +set TKDNN_CALIB_LABEL_PATH=..\demo\COCO_val2017\all_labels.txt +set TKDNN_CALIB_IMG_PATH=..\demo\COCO_val2017\all_images.txt +del /f yolo4tiny_int8.rt # be sure to delete(or move) old tensorRT files +.\test_yolo4tiny.exe # run the yolo test (is slow) +.\demo.exe yolo4tiny_int8.rt ..\demo\yolo_test.mp4 y + +``` + + + + + + + + ## References -- 2.52.0 From f3d159143025672d5cd1bc91a89370dac1d902c2 Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Thu, 25 Mar 2021 20:46:12 +0530 Subject: [PATCH 115/228] ReadMe.md windows changes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 242220a..f685b58 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001 - [Existing tests and supported networks](#existing-tests-and-supported-networks) - [References](#references) - [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows) - - [Dependencies](#dependencies) + - [Dependencies-Windows](#dependencies-windows) - [Compiling tkDNN on Windows](#tkdnn-windows-compile) - [Run the demo on Windows](#run-the-demo-on-windows) - [FP16 interference windows](#fp16-windows) @@ -364,7 +364,7 @@ This demo also creates a json file named ```net_name_COCO_res.json``` containing ##tkDNN on Windows 10 (experimental) -### Dependencies +### Dependencies-Windows This branch should work on every NVIDIA GPU supported in windows with the following dependencies: * WINDOWS 10 1803 or HIGHER -- 2.52.0 From f12ec3c935ce4fdcb45c966e9b4f753454d630b9 Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Fri, 26 Mar 2021 12:18:46 +0530 Subject: [PATCH 116/228] added yolo4x and yolo4-csp from the github repo and download file corrections --- include/tkDNN/Layer.h | 7 +- include/tkDNN/NetworkRT.h | 1 + .../tkDNN/pluginsRT/ActivationLogisticRT.h | 60 + include/tkDNN/pluginsRT/YoloRT.h | 29 +- src/Activation.cpp | 6 +- src/DarknetParser.cpp | 1 + src/NetworkRT.cpp | 21 +- src/Yolo.cpp | 21 +- tests/darknet/cfg/yolo4-csp.cfg | 1279 +++++++++++++++++ tests/darknet/cfg/yolo4x.cfg | 21 +- tests/darknet/yolo4-csp.cpp | 36 + tests/darknet/yolo4x.cpp | 2 +- 12 files changed, 1444 insertions(+), 40 deletions(-) create mode 100644 include/tkDNN/pluginsRT/ActivationLogisticRT.h create mode 100644 tests/darknet/cfg/yolo4-csp.cfg create mode 100644 tests/darknet/yolo4-csp.cpp diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 25c4565..e097372 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -19,6 +19,7 @@ enum layerType_t { LAYER_ACTIVATION_CRELU, LAYER_ACTIVATION_LEAKY, LAYER_ACTIVATION_MISH, + LAYER_ACTIVATION_LOGISTIC, LAYER_FLATTEN, LAYER_RESHAPE, LAYER_MULADD, @@ -68,6 +69,7 @@ public: case LAYER_ACTIVATION_CRELU: return "ActivationCReLU"; case LAYER_ACTIVATION_LEAKY: return "ActivationLeaky"; case LAYER_ACTIVATION_MISH: return "ActivationMish"; + case LAYER_ACTIVATION_LOGISTIC: return "ActivationLogistic"; case LAYER_FLATTEN: return "Flatten"; case LAYER_RESHAPE: return "Reshape"; case LAYER_MULADD: return "MulAdd"; @@ -212,7 +214,8 @@ public: typedef enum { ACTIVATION_ELU = 100, ACTIVATION_LEAKY = 101, - ACTIVATION_MISH = 102 + ACTIVATION_MISH = 102, + ACTIVATION_LOGISTIC = 103 } tkdnnActivationMode_t; /** @@ -233,6 +236,8 @@ public: return LAYER_ACTIVATION_LEAKY; else if (act_mode == ACTIVATION_MISH) return LAYER_ACTIVATION_MISH; + else if (act_mode == ACTIVATION_LOGISTIC) + return LAYER_ACTIVATION_LOGISTIC; else return LAYER_ACTIVATION; }; diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index a50cd8c..b39360c 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -27,6 +27,7 @@ using namespace nvinfer1; #include "pluginsRT/ActivationLeakyRT.h" #include "pluginsRT/ActivationReLUCeilingRT.h" #include "pluginsRT/ActivationMishRT.h" +#include "pluginsRT/ActivationLogisticRT.h" #include "pluginsRT/ReorgRT.h" #include "pluginsRT/RegionRT.h" #include "pluginsRT/RouteRT.h" diff --git a/include/tkDNN/pluginsRT/ActivationLogisticRT.h b/include/tkDNN/pluginsRT/ActivationLogisticRT.h new file mode 100644 index 0000000..83f62ff --- /dev/null +++ b/include/tkDNN/pluginsRT/ActivationLogisticRT.h @@ -0,0 +1,60 @@ +#include +#include "../kernels.h" + +class ActivationLogisticRT : public IPlugin { + +public: + ActivationLogisticRT() { + + + } + + ~ActivationLogisticRT(){ + + } + + int getNbOutputs() const override { + return 1; + } + + Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override { + return inputs[0]; + } + + void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override { + size = 1; + for(int i=0; i(inputs[0]), + reinterpret_cast(outputs[0]), batchSize*size, stream); + return 0; + } + + + virtual size_t getSerializationSize() override { + return 1*sizeof(int); + } + + virtual void serialize(void* buffer) override { + char *buf = reinterpret_cast(buffer); + tk::dnn::writeBUF(buf, size); + } + + int size; +}; \ No newline at end of file diff --git a/include/tkDNN/pluginsRT/YoloRT.h b/include/tkDNN/pluginsRT/YoloRT.h index 451d99f..0dd26e1 100644 --- a/include/tkDNN/pluginsRT/YoloRT.h +++ b/include/tkDNN/pluginsRT/YoloRT.h @@ -64,20 +64,23 @@ public: checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream)); - for (int b = 0; b < batchSize; ++b){ - for(int n = 0; n < n_masks; ++n){ - int index = entry_index(b, n*w*h, 0); - if (new_coords == 1) - activationLOGISTICForward(srcData + index, dstData + index, 4*w*h, stream); //x,y,w,h - else - activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream); //x,y - if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); - - index = entry_index(b, n*w*h, 4); - activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream); - } - } + for (int b = 0; b < batchSize; ++b){ + for(int n = 0; n < n_masks; ++n){ + int index = entry_index(b, n*w*h, 0); + if (new_coords == 1){ + if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); + } + else{ + activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream); //x,y + + if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); + + index = entry_index(b, n*w*h, 4); + activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream); + } + } + } //std::cout<<"YOLO END\n"; return 0; diff --git a/src/Activation.cpp b/src/Activation.cpp index 28c7624..4219271 100644 --- a/src/Activation.cpp +++ b/src/Activation.cpp @@ -52,7 +52,11 @@ dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) { else if(act_mode == ACTIVATION_MISH) { activationMishForward(srcData, dstData, dim.tot()); - } else { + } + else if(act_mode == ACTIVATION_LOGISTIC) { + activationLOGISTICForward(srcData, dstData, dim.tot()); + + }else { dnnType alpha = dnnType(1); dnnType beta = dnnType(0); checkCUDNN( cudnnActivationForward(net->cudnnHandle, diff --git a/src/DarknetParser.cpp b/src/DarknetParser.cpp index 7b5410c..69b6b29 100644 --- a/src/DarknetParser.cpp +++ b/src/DarknetParser.cpp @@ -187,6 +187,7 @@ namespace tk { namespace dnn { if(f.activation == "relu") act = tkdnnActivationMode_t(CUDNN_ACTIVATION_RELU); else if(f.activation == "leaky") act = tk::dnn::ACTIVATION_LEAKY; else if(f.activation == "mish") act = tk::dnn::ACTIVATION_MISH; + else if(f.activation == "logistic") act = tk::dnn::ACTIVATION_LOGISTIC; else { FatalError("activation not supported: " + f.activation); } netLayers[netLayers.size()-1] = new tk::dnn::Activation(net, act); }; diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 29df411..1e0b062 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -229,7 +229,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) { return convert_layer(input, (Conv2d*) l); if(type == LAYER_POOLING) return convert_layer(input, (Pooling*) l); - if(type == LAYER_ACTIVATION || type == LAYER_ACTIVATION_CRELU || type == LAYER_ACTIVATION_LEAKY || type == LAYER_ACTIVATION_MISH) + if(type == LAYER_ACTIVATION || type == LAYER_ACTIVATION_CRELU || type == LAYER_ACTIVATION_LEAKY || type == LAYER_ACTIVATION_MISH || type == LAYER_ACTIVATION_LOGISTIC) return convert_layer(input, (Activation*) l); if(type == LAYER_SOFTMAX) return convert_layer(input, (Softmax*) l); @@ -424,6 +424,12 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { checkNULL(lRT); return lRT; } + else if(l->act_mode == ACTIVATION_LOGISTIC) { + IPlugin *plugin = new ActivationLogisticRT(); + IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); + checkNULL(lRT); + return lRT; + } else { FatalError("this Activation mode is not yet implemented"); return NULL; @@ -660,6 +666,11 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa assert(buf == bufCheck + serialLength); return a; } + if(name.find("ActivationLogistic") == 0) { + ActivationLogisticRT *a = new ActivationLogisticRT(); + a->size = readBUF(buf); + return a; + } if(name.find("ActivationCReLU") == 0) { float activationReluTemp = readBUF(buf); //ActivationReLUCeiling *a = new ActivationReLUCeiling(readBUF(buf)); @@ -784,17 +795,9 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa float nms_thresh_temp = readBUF(buf); int nms_kind_temp = readBUF(buf); int new_coords_temp = readBUF(buf); - std::cout << classes_temp << ":" << num_temp << ":" << ":" << n_masks_temp << ":" << nms_thresh_temp << ":" << nms_kind_temp << ":" << new_coords_temp << std::endl; YoloRT *r = new YoloRT(classes_temp,num_temp,nullptr,n_masks_temp,scale_xy_temp,nms_thresh_temp,nms_kind_temp,new_coords_temp); - /* std::cout << "classes : " << r->classes; - std::cout << "num : " << r->num; - std::cout << "n_masks : " << r->n_masks; - std::cout << "scalexy : " << r->scaleXY; - std::cout << "nms_thresh : " << r->nms_thresh; - std::cout << "nms_kind : " << r->nms_kind; - std::cout << "new_coords : " << r->new_coords;*/ r->c = readBUF(buf); diff --git a/src/Yolo.cpp b/src/Yolo.cpp index 6d0b546..1eb7843 100644 --- a/src/Yolo.cpp +++ b/src/Yolo.cpp @@ -73,8 +73,8 @@ Yolo::box get_yolo_box(float *x, float *biases, int n, int index, int i, int j, b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h; } else{ - b.x = (i + x[index + 0 * stride] * 2 - 0.5) / lw; - b.y = (j + x[index + 1 * stride] * 2 - 0.5) / lh; + b.x = (i + x[index + 0 * stride] ) / lw; + b.y = (j + x[index + 1 * stride] ) / lh; b.w = x[index + 2 * stride] * x[index + 2 * stride] * 4 * biases[2 * n] / w; b.h = x[index + 3 * stride] * x[index + 3 * stride] * 4 * biases[2 * n + 1] / h; } @@ -88,15 +88,18 @@ dnnType* Yolo::infer(dataDim_t &dim, dnnType* srcData) { for (int b = 0; b < dim.n; ++b){ for(int n = 0; n < n_masks; ++n){ int index = entry_index(b, n*dim.w*dim.h, 0, classes, input_dim, output_dim); - if (new_coords == 1) - activationLOGISTICForward(srcData + index, dstData + index, 4*dim.w*dim.h); - else + std::cout<<"new_coords"<scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); + } + else{ activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h); - if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); - - index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim); - activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h); + if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1); + + index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim); + activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h); + } } } diff --git a/tests/darknet/cfg/yolo4-csp.cfg b/tests/darknet/cfg/yolo4-csp.cfg new file mode 100644 index 0000000..887898e --- /dev/null +++ b/tests/darknet/cfg/yolo4-csp.cfg @@ -0,0 +1,1279 @@ +[net] +# Testing +#batch=1 +#subdivisions=1 +# Training +batch=64 +subdivisions=8 +width=512 +height=512 +channels=3 +momentum=0.949 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=1000 +max_batches = 500500 +policy=steps +steps=400000,450000 +scales=.1,.1 + +mosaic=1 + +letter_box=1 + +ema_alpha=0.9998 + +#optimized_memory=1 + +#23:104x104 54:52x52 85:26x26 104:13x13 for 416 + + + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=mish + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +#[route] +#layers = -2 + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +#[route] +#layers = -1,-7 + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-10 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-16 + +[convolutional] +batch_normalize=1 +filters=1024 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +### SPP ### +[maxpool] +stride=1 +size=5 + +[route] +layers=-2 + +[maxpool] +stride=1 +size=9 + +[route] +layers=-4 + +[maxpool] +stride=1 +size=13 + +[route] +layers=-1,-3,-5,-6 +### End SPP ### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1, -13 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 79 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1, -6 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 48 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=128 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=128 +activation=mish + +[route] +layers = -1, -6 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=logistic + + +[yolo] +mask = 0,1,2 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=0 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=4.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=5 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1, -20 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1,-6 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=logistic + + +[yolo] +mask = 3,4,5 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=1.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=5 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1, -49 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1,-6 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=logistic + + +[yolo] +mask = 6,7,8 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=0.4 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=2 \ No newline at end of file diff --git a/tests/darknet/cfg/yolo4x.cfg b/tests/darknet/cfg/yolo4x.cfg index 89f2564..f6604f6 100644 --- a/tests/darknet/cfg/yolo4x.cfg +++ b/tests/darknet/cfg/yolo4x.cfg @@ -5,8 +5,8 @@ # Training batch=64 subdivisions=8 -width=672 -height=672 +width=640 +height=640 channels=3 momentum=0.949 decay=0.0005 @@ -15,7 +15,7 @@ saturation = 1.5 exposure = 1.5 hue=.1 -learning_rate=0.00261 +learning_rate=0.001 burn_in=1000 max_batches = 500500 policy=steps @@ -26,6 +26,8 @@ mosaic=1 letter_box=1 +#optimized_memory=1 + [convolutional] batch_normalize=1 filters=32 @@ -1131,6 +1133,7 @@ size=1 stride=1 pad=1 activation=mish +stopbackward=800 ########################## @@ -1147,7 +1150,7 @@ size=1 stride=1 pad=1 filters=255 -activation=linear +activation=logistic [yolo] @@ -1156,6 +1159,7 @@ anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 4 classes=80 num=9 jitter=.1 +scale_x_y = 2.0 objectness_smooth=0 ignore_thresh = .7 truth_thresh = 1 @@ -1169,6 +1173,7 @@ iou_loss=ciou nms_kind=diounms beta_nms=0.6 new_coords=1 +max_delta=5 [route] layers = -4 @@ -1275,7 +1280,7 @@ size=1 stride=1 pad=1 filters=255 -activation=linear +activation=logistic [yolo] @@ -1284,6 +1289,7 @@ anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 4 classes=80 num=9 jitter=.1 +scale_x_y = 2.0 objectness_smooth=1 ignore_thresh = .7 truth_thresh = 1 @@ -1297,6 +1303,7 @@ iou_loss=ciou nms_kind=diounms beta_nms=0.6 new_coords=1 +max_delta=5 [route] layers = -4 @@ -1403,7 +1410,7 @@ size=1 stride=1 pad=1 filters=255 -activation=linear +activation=logistic [yolo] @@ -1412,6 +1419,7 @@ anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 4 classes=80 num=9 jitter=.1 +scale_x_y = 2.0 objectness_smooth=1 ignore_thresh = .7 truth_thresh = 1 @@ -1425,3 +1433,4 @@ iou_loss=ciou nms_kind=diounms beta_nms=0.6 new_coords=1 +max_delta=2 \ No newline at end of file diff --git a/tests/darknet/yolo4-csp.cpp b/tests/darknet/yolo4-csp.cpp new file mode 100644 index 0000000..3802a9a --- /dev/null +++ b/tests/darknet/yolo4-csp.cpp @@ -0,0 +1,36 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4-csp"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer144_out.bin", + bin_path + "/debug/layer159_out.bin", + bin_path + "/debug/layer174_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4-csp.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/AfzHE4BfTeEm2gH/download"); + + + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} \ No newline at end of file diff --git a/tests/darknet/yolo4x.cpp b/tests/darknet/yolo4x.cpp index b9ad003..8df1aef 100644 --- a/tests/darknet/yolo4x.cpp +++ b/tests/darknet/yolo4x.cpp @@ -17,7 +17,7 @@ int main() { std::string wgs_path = bin_path + "/layers"; std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4x.cfg"; std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/BLPpiAigZJLorQD/download"); + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download"); -- 2.52.0 From 7018d163ed2741bc1f095f9f129368beecdee567 Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Fri, 26 Mar 2021 19:37:44 +0530 Subject: [PATCH 117/228] python download file --- scripts/download_validation.py | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 scripts/download_validation.py diff --git a/scripts/download_validation.py b/scripts/download_validation.py new file mode 100644 index 0000000..e49f2ab --- /dev/null +++ b/scripts/download_validation.py @@ -0,0 +1,40 @@ +import os +from pathlib import Path +import urllib.request as dowReq +import zipfile + +val = input("Enter BDD or COCO :") +if(val == "COCO"): + url = "https://cloud.hipert.unimore.it/s/LNxBDk4wzqXPL8c/download" + lib = "..\demo\COCO_val2017" + lib_zip = "COCO_val2017.zip" +elif(val == "BDD"): + url = "https://cloud.hipert.unimore.it/s/bikqk3FzCq2tg4D/download" + lib = "..\demo\BDD100k_val" + lib_zip = "BDD100k_val.zip" + +dowReq.urlretrieve(url,lib_zip) + +with zipfile.ZipFile(lib_zip,'r') as zip_ref: + zip_ref.extractall(lib) + +labelFolder = lib + "\labels" +imageFolder = lib + "\images" + +file1 = open(".\\..\\demo\\all_labels.txt","a") +path1 = os.path.realpath(labelFolder) +for file in os.listdir(labelFolder): + valTemp = path1 + "\\" + file + valTemp = valTemp + " \n" + file1.write(valTemp) +file1.close() + +file2 = open(".\\..\\demo\\all_images.txt","a") +path2 = os.path.realpath(imageFolder) +for file in os.listdir(imageFolder): + pathtemp = path2 + "\\" + file + pathtemp = pathtemp + " \n" + file2.write(pathtemp) +file2.close() + +print("Completed") \ No newline at end of file -- 2.52.0 From 5f3ab1472c8c02be43039abc273bcd6afc9f823d Mon Sep 17 00:00:00 2001 From: Harshvardhan Chandirasekar Date: Fri, 26 Mar 2021 15:09:28 +0100 Subject: [PATCH 118/228] Update download_validation.py --- scripts/download_validation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/download_validation.py b/scripts/download_validation.py index e49f2ab..e531b4f 100644 --- a/scripts/download_validation.py +++ b/scripts/download_validation.py @@ -1,5 +1,4 @@ import os -from pathlib import Path import urllib.request as dowReq import zipfile @@ -37,4 +36,4 @@ for file in os.listdir(imageFolder): file2.write(pathtemp) file2.close() -print("Completed") \ No newline at end of file +print("Completed") -- 2.52.0 From 2e92944f1da276ec31eeb08bd99cf7e78679b23c Mon Sep 17 00:00:00 2001 From: hchandirasekar Date: Wed, 31 Mar 2021 03:44:51 -0700 Subject: [PATCH 119/228] Opencv cuda fix --- demo/demo/demo.cpp | 2 +- include/tkDNN/DetectionNN.h | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index f97ead9..59bb1a2 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -25,7 +25,7 @@ int main(int argc, char *argv[]) { std::string net = "yolo4tiny_fp32.rt"; if(argc > 1) net = argv[1]; - std::string input = "../demo/yolo_test.mp4"; + std::string input = "..\..\..\demo\yolo_test.mp4"; if(argc > 2) input = argv[2]; char ntype = 'y'; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index fb29be1..b1266e0 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -6,8 +6,6 @@ #include #ifdef __linux__ #include -#elif _WIN32 -#include #endif #include @@ -19,7 +17,7 @@ #include "tkdnn.h" -// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. +#define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. #ifdef OPENCV_CUDACONTRIB #include -- 2.52.0 From cc594f09efdbf0e7c5a33e4e4c40ea24ac2e7048 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Thu, 8 Apr 2021 00:28:00 +0530 Subject: [PATCH 120/228] merge from gitlab --- include/tkDNN/NetworkRT.h | 10 ++++------ include/tkDNN/utils.h | 14 -------------- src/NetworkRT.cpp | 37 ++++--------------------------------- src/utils.cpp | 3 +-- 4 files changed, 9 insertions(+), 55 deletions(-) diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index b39360c..66892f5 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -55,16 +55,14 @@ class NetworkRT { public: nvinfer1::DataType dtRT; - //nvinfer1::IBuilder *builderRT; - std::unique_ptr builderRT; - //nvinfer1::IRuntime *runtimeRT; - std::unique_ptr runtimeRT; + nvinfer1::IBuilder *builderRT; + nvinfer1::IRuntime *runtimeRT; nvinfer1::INetworkDefinition *networkRT; #if NV_TENSORRT_MAJOR >= 6 nvinfer1::IBuilderConfig *configRT; #endif - std::shared_ptr engineRT; - //nvinfer1::ICudaEngine *engineRT; + + nvinfer1::ICudaEngine *engineRT; nvinfer1::IExecutionContext *contextRT; const static int MAX_BUFFERS_RT = 10; diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index ce4a617..eeef3c2 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -14,9 +14,6 @@ #ifdef __linux__ #include -#elif _WIN32 -#define NOMINMAX -#include #endif #include @@ -110,17 +107,6 @@ double t_ns = time_ms.count(); FatalError(_error.str()); \ } \ } -struct InferDeleter -{ - template - void operator()(T* obj) const - { - if (obj) - { - obj->destroy(); - } - } -}; typedef enum { ERROR_CUDNN = 2, diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 1e0b062..b5005db 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -33,8 +33,7 @@ NetworkRT::NetworkRT(Network *net, const char *name) { float(NV_TENSORRT_PATCH)/100; std::cout<<"New NetworkRT (TensorRT v"<(createInferBuilder(loggerRT)); - //builderRT = createInferBuilder(loggerRT); + builderRT = createInferBuilder(loggerRT); std::cout<<"Float16 support: "<platformHasFastFp16()<<"\n"; std::cout<<"Int8 support: "<platformHasFastInt8()<<"\n"; #if NV_TENSORRT_MAJOR >= 5 @@ -138,8 +137,7 @@ NetworkRT::NetworkRT(Network *net, const char *name) { printCudaMemUsage(); std::cout<<"Building tensorRT cuda engine...\n"; #if NV_TENSORRT_MAJOR >= 6 - //engineRT = builderRT->buildEngineWithConfig(*networkRT, *configRT); - engineRT = std::shared_ptr(builderRT->buildEngineWithConfig(*networkRT,*configRT),InferDeleter()); + engineRT = builderRT->buildEngineWithConfig(*networkRT, *configRT); #else //engineRT = builderRT->buildCudaEngine(*networkRT); engineRT = std::shared_ptr(builderRT->buildCudaEngine(*networkRT)); @@ -637,10 +635,8 @@ bool NetworkRT::deserialize(const char *filename) { } pluginFactory = new PluginFactory(); - //runtimeRT = createInferRuntime(loggerRT); - runtimeRT = std::unique_ptr(createInferRuntime(loggerRT)); - //engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size, (IPluginFactory *) pluginFactory); - engineRT = std::shared_ptr(runtimeRT->deserializeCudaEngine(gieModelStream,size,(IPluginFactory*)pluginFactory),InferDeleter()); + runtimeRT = createInferRuntime(loggerRT); + engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size, (IPluginFactory *) pluginFactory); //if (gieModelStream) delete [] gieModelStream; return true; @@ -673,7 +669,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa } if(name.find("ActivationCReLU") == 0) { float activationReluTemp = readBUF(buf); - //ActivationReLUCeiling *a = new ActivationReLUCeiling(readBUF(buf)); ActivationReLUCeiling* a = new ActivationReLUCeiling(activationReluTemp); a->size = readBUF(buf); assert(buf == bufCheck + serialLength); @@ -684,9 +679,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa int classesTemp = readBUF(buf); int coordsTemp = readBUF(buf); int numTemp = readBUF(buf); - /*RegionRT *r = new RegionRT(readBUF(buf), //classes - readBUF(buf), //coords - readBUF(buf)); //num8*/ RegionRT* r = new RegionRT(classesTemp, coordsTemp, numTemp); r->c = readBUF(buf); @@ -698,7 +690,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa if(name.find("Reorg") == 0) { int strideTemp = readBUF(buf); - //ReorgRT *r = new ReorgRT(readBUF(buf)); //stride ReorgRT *r = new ReorgRT(strideTemp); r->c = readBUF(buf); r->h = readBUF(buf); @@ -723,15 +714,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa } if(name.find("Pooling") == 0) { - /* MaxPoolFixedSizeRT *r = new MaxPoolFixedSizeRT( readBUF(buf), //c - readBUF(buf), //h - readBUF(buf), //w - readBUF(buf), //n - readBUF(buf), //strideH - readBUF(buf), //strideW - readBUF(buf), //winSize - readBUF(buf)); //padding*/ - int cTemp = readBUF(buf); int hTemp = readBUF(buf); int wTemp = readBUF(buf); @@ -747,9 +729,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa } if(name.find("Resize") == 0) { - /*ResizeLayerRT *r = new ResizeLayerRT(readBUF(buf), //o_c - readBUF(buf), //o_h - readBUF(buf)); //o_w*/ int o_cTemp = readBUF(buf); int o_hTemp = readBUF(buf); int o_wTemp = readBUF(buf); @@ -822,7 +801,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa return r; } if(name.find("Upsample") == 0) { - //UpsampleRT *r = new UpsampleRT(readBUF(buf)); //stride int strideTemp = readBUF(buf); UpsampleRT* r = new UpsampleRT(strideTemp); r->c = readBUF(buf); @@ -833,7 +811,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa } if(name.find("Route") == 0) { - //RouteRT *r = new RouteRT(readBUF(buf),readBUF(buf)); int groupsTemp = readBUF(buf); int group_idTemp = readBUF(buf); RouteRT* r = new RouteRT(groupsTemp, group_idTemp); @@ -848,12 +825,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa } if(name.find("Deformable") == 0) { - /*DeformableConvRT *r = new DeformableConvRT(readBUF(buf), readBUF(buf), readBUF(buf), - readBUF(buf), readBUF(buf), readBUF(buf), - readBUF(buf), readBUF(buf), - readBUF(buf),readBUF(buf),readBUF(buf),readBUF(buf), - readBUF(buf),readBUF(buf),readBUF(buf),readBUF(buf), - nullptr); */ int chuck_dimTemp = readBUF(buf); int khTemp = readBUF(buf); int kwTemp = readBUF(buf); diff --git a/src/utils.cpp b/src/utils.cpp index 52775df..fa6458f 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -203,8 +203,7 @@ void getMemUsage(double& vm_usage_kb, double& resident_set_kb){ #ifdef __linux__ long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages #elif _WIN32 -SYSTEM_INFO sysInfo; - long page_size_kb = sysInfo.dwPageSize/1024; + long page_size_kb = 4096/1024; #endif vm_usage_kb = vsize / 1024.0; -- 2.52.0 From 37b2a5bd9856b8a6becb8ac696c95f70eaef20e9 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Thu, 8 Apr 2021 00:57:58 +0530 Subject: [PATCH 121/228] minor modifications --- demo/demo/map.cpp | 2 -- include/tkDNN/ImuOdom.h | 1 - include/tkDNN/Int8BatchStream.h | 2 -- 3 files changed, 5 deletions(-) diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 8e363ee..58490e5 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -4,8 +4,6 @@ #include /* srand, rand */ #ifdef __linux__ #include -#elif _WIN32 -#include #endif #include diff --git a/include/tkDNN/ImuOdom.h b/include/tkDNN/ImuOdom.h index fa870f3..d5429a8 100644 --- a/include/tkDNN/ImuOdom.h +++ b/include/tkDNN/ImuOdom.h @@ -7,7 +7,6 @@ #elif _WIN32 #define _USE_MATH_DEFINES #include -#include #endif #include diff --git a/include/tkDNN/Int8BatchStream.h b/include/tkDNN/Int8BatchStream.h index 7d2cef5..c39a11c 100644 --- a/include/tkDNN/Int8BatchStream.h +++ b/include/tkDNN/Int8BatchStream.h @@ -14,8 +14,6 @@ #include #ifdef __linux__ #include -#elif _WIN32 -#include #endif #include -- 2.52.0 From 1de804f98dd67e66e6893e0c64749befdb9b6371 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Fri, 9 Apr 2021 13:17:41 +0530 Subject: [PATCH 122/228] Code cleanup and readme fixes --- CMakeLists.txt | 6 ++--- README.md | 39 +++++++++++++++++----------- demo/demo/demo.cpp | 7 ++++- include/tkDNN/DetectionNN.h | 2 +- include/tkDNN/pluginsRT/UpsampleRT.h | 2 -- include/tkDNN/pluginsRT/YoloRT.h | 1 - scripts/download_validation.py | 4 +-- src/NetworkRT.cpp | 2 +- 8 files changed, 37 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b7ed63..c27e519 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,11 +3,11 @@ cmake_minimum_required(VERSION 3.5) project (tkDNN) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) if(UNIX) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -fPIC -Wno-deprecated-declarations -Wno-unused-variable") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable -g ") endif() if(WIN32) -set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_FLAGS "/O1 /FS /EHsc") +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_FLAGS "/O2 /FS /EHsc") set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) endif(WIN32) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN) diff --git a/README.md b/README.md index f685b58..cbc65b0 100644 --- a/README.md +++ b/README.md @@ -80,12 +80,13 @@ Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001 - [mAP demo](#map-demo) - [Existing tests and supported networks](#existing-tests-and-supported-networks) - [References](#references) - - [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows) + - [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows-10-experimental) - [Dependencies-Windows](#dependencies-windows) - - [Compiling tkDNN on Windows](#tkdnn-windows-compile) + - [Compiling tkDNN on Windows](#compiling-tkdnn-on-windows) - [Run the demo on Windows](#run-the-demo-on-windows) - - [FP16 interference windows](#fp16-windows) - - [INT8 interference windows](#int8-windows) + - [FP16 inference windows](#fp16-inference-windows) + - [INT8 inference windows](#int8-inference-windows) + - [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows) @@ -362,26 +363,31 @@ This demo also creates a json file named ```net_name_COCO_res.json``` containing | yolo4tiny | Yolov4 tiny 9 | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | | yolo4x | Yolov4x-mish 9 | [COCO 2017](http://cocodataset.org/) | 80 | 672x672 | [weights](https://cloud.hipert.unimore.it/s/BLPpiAigZJLorQD/download) | -##tkDNN on Windows 10 (experimental) +### tkDNN on Windows 10 (experimental) ### Dependencies-Windows This branch should work on every NVIDIA GPU supported in windows with the following dependencies: * WINDOWS 10 1803 or HIGHER -* CUDA 10.0 (Recommended CUDA 11.0 +) -* CUDNN 7.6 (Recommended CUDNN 8.0.0 +) -* TENSORRT 6.0.1 (Recommended TENSORRT 7.1 +) -* OPENCV 3.4 (Recommended OPENCV 4.2.0 +) -* MSVC 16.7 (Recommended MSVC 16.8/16.9) -* YAML-CPP 0.5.2 +* CUDA 10.0 (Recommended CUDA 11.2 ) +* CUDNN 7.6 (Recommended CUDNN 8.1.1 ) +* TENSORRT 6.0.1 (Recommended TENSORRT 7.2.3.4 ) +* OPENCV 3.4 (Recommended OPENCV 4.2.0 ) +* MSVC 16.7 +* YAML-CPP * EIGEN3 * 7ZIP (ADD TO PATH) * NINJA 1.10 + All the above mentioned dependencies except 7ZIP can be installed using Microsoft's [VCPKG](https://github.com/microsoft/vcpkg.git) . After bootstrapping VCPKG the dependencies can be built and installed using the following command : -```vcpkg.exe install opencv4[tbb,jpeg,tiff,opengl,openmp,png,ffmpeg]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build``` +``` +opencv4(normal) - vcpkg.exe install opencv4[tbb,jpeg,tiff,opengl,openmp,png,ffmpeg,eigen]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build + +opencv4(cuda) - vcpkg.exe install opencv4[cuda,nonfree,contrib,eigen,tbb,jpeg,tiff,opengl,openmp,png,ffmpeg]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build +``` After VCPKG finishes building and installing all the packages delete C:\temp_vcpkg_build and add C:\opt\x64-windows\bin and C:\opt\x64-windows\debug\bin to path @@ -411,7 +417,7 @@ Once the rt file has been successfully create,run the demo using the following c ``` For general info on more demo paramters,check Run the demo section on top -### FP16 interference windows +### FP16 inference windows This is an untested feature on windows.To run the object detection demo with FP16 interference follow the below steps(example with yolo4tiny): ``` @@ -421,7 +427,7 @@ del /f yolo4tiny_fp16.rt .\demo.exe yolo4tiny_fp16.rt ..\demo\yolo_test.mp4 ``` -### INT8 interference windows +### INT8 inference windows To run object detection demo with INT8 (example with yolo4tiny): ``` set TKDNN_MODE=INT8 @@ -433,10 +439,13 @@ del /f yolo4tiny_int8.rt # be sure to delete(or move) old tensorRT files ``` +### Known issues with tkDNN on Windows +Mobilenet and Centernet demos work properly only when built with msvc 16.7 in Release Mode,when built in debug mode for the mentioned networks one might encounter opencv assert errors +All Darknet models work properly with demo using MSVC version(16.7-16.9) - +It is recommended to use Nvidia Driver(465+),Cuda unknown errors have been observed when using older drivers on pascal(SM 61) devices. diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 59bb1a2..317a574 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -25,7 +25,12 @@ int main(int argc, char *argv[]) { std::string net = "yolo4tiny_fp32.rt"; if(argc > 1) net = argv[1]; - std::string input = "..\..\..\demo\yolo_test.mp4"; + #ifdef __linux__ + std::string input = "../demo/yolo_test.mp4"; + #elif _WIN32 + std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + #endif + if(argc > 2) input = argv[2]; char ntype = 'y'; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index b1266e0..a8c81f7 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -17,7 +17,7 @@ #include "tkdnn.h" -#define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. +//#define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. #ifdef OPENCV_CUDACONTRIB #include diff --git a/include/tkDNN/pluginsRT/UpsampleRT.h b/include/tkDNN/pluginsRT/UpsampleRT.h index 5350b7e..a11d7b4 100644 --- a/include/tkDNN/pluginsRT/UpsampleRT.h +++ b/include/tkDNN/pluginsRT/UpsampleRT.h @@ -59,8 +59,6 @@ public: tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); - std::cout << "Upsample Serialization SIze" << getSerializationSize() << std::endl; - assert(buf == a + getSerializationSize()); } diff --git a/include/tkDNN/pluginsRT/YoloRT.h b/include/tkDNN/pluginsRT/YoloRT.h index 0dd26e1..2911869 100644 --- a/include/tkDNN/pluginsRT/YoloRT.h +++ b/include/tkDNN/pluginsRT/YoloRT.h @@ -120,7 +120,6 @@ public: tk::dnn::writeBUF(buf, tmp[j]); } } - std::cout << getSerializationSize() << std::endl; assert(buf == a + getSerializationSize()); } diff --git a/scripts/download_validation.py b/scripts/download_validation.py index e531b4f..0e3b1d4 100644 --- a/scripts/download_validation.py +++ b/scripts/download_validation.py @@ -24,7 +24,7 @@ file1 = open(".\\..\\demo\\all_labels.txt","a") path1 = os.path.realpath(labelFolder) for file in os.listdir(labelFolder): valTemp = path1 + "\\" + file - valTemp = valTemp + " \n" + valTemp = valTemp + '\n' file1.write(valTemp) file1.close() @@ -32,7 +32,7 @@ file2 = open(".\\..\\demo\\all_images.txt","a") path2 = os.path.realpath(imageFolder) for file in os.listdir(imageFolder): pathtemp = path2 + "\\" + file - pathtemp = pathtemp + " \n" + pathtemp = pathtemp + '\n' file2.write(pathtemp) file2.close() diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index b5005db..5dc8ee0 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -648,7 +648,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa const char * buf = reinterpret_cast(serialData),*bufCheck = buf; std::string name(layerName); - std::cout< Date: Fri, 9 Apr 2021 13:20:44 +0530 Subject: [PATCH 123/228] Update CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c27e519..d3a89f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.5) project (tkDNN) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) if(UNIX) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable -g ") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable ") endif() if(WIN32) set(CMAKE_CXX_STANDARD 11) -- 2.52.0 From 39323ca8d3ac02a8a1e74ede6c851e57546f18ab Mon Sep 17 00:00:00 2001 From: Harshvardhan Chandirasekar <43143075+perseusdg@users.noreply.github.com> Date: Wed, 14 Apr 2021 17:22:26 +0530 Subject: [PATCH 124/228] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cbc65b0..7fb389b 100644 --- a/README.md +++ b/README.md @@ -388,6 +388,7 @@ opencv4(normal) - vcpkg.exe install opencv4[tbb,jpeg,tiff,opengl,openmp,png,ffmp opencv4(cuda) - vcpkg.exe install opencv4[cuda,nonfree,contrib,eigen,tbb,jpeg,tiff,opengl,openmp,png,ffmpeg]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build ``` +To build opencv4 with cuda and cudnn version corresponding to your cuda version,vcpkg's cudnn portfile needs to be modified by adding ```$ENV{CUDA_PATH}``` at lines 16 and 17 in the portfile.cmake After VCPKG finishes building and installing all the packages delete C:\temp_vcpkg_build and add C:\opt\x64-windows\bin and C:\opt\x64-windows\debug\bin to path @@ -395,7 +396,7 @@ After VCPKG finishes building and installing all the packages delete C:\temp_vcp tkDNN is built with cmake(3.15+) on windows along with ninja.Msbuild and NMake Makefiles are drastically slower when compiling the library compared to windows ``` -git clone https://git.hipert.unimore.it/research-cv-chandirasekar/tkdnn-windows.git +git clone https://github.com/ceccocats/tkDNN.git cd tkdnn-windows mkdir build cd build @@ -416,6 +417,7 @@ Once the rt file has been successfully create,run the demo using the following c .\demo.exe yolo4tiny_fp32.rt ..\demo\yolo_test.mp4 y ``` For general info on more demo paramters,check Run the demo section on top + To run the test_all_tests.sh on windows,use git bash or msys2 ### FP16 inference windows -- 2.52.0 From be6ad27c11f85481576037658fda7e97005340c9 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Thu, 29 Apr 2021 11:13:24 +0200 Subject: [PATCH 125/228] Batch size > 1 for the 3D demo. This commit lets to use differtent batch size for 3D CenterNet and CenterTrack. Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 71 ++++-- include/tkDNN/CenternetDetection3D.h | 12 +- include/tkDNN/CenternetDetection3DTrack.h | 13 +- include/tkDNN/DetectionNN3D.h | 83 +++--- src/CenternetDetection3D.cpp | 101 ++++---- src/CenternetDetection3DTrack.cpp | 291 +++++++++++----------- 6 files changed, 309 insertions(+), 262 deletions(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 4286faa..c35ac51 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -1,7 +1,7 @@ #include #include #include /* srand, rand */ -#include +//#include #include #include "CenternetDetection3D.h" @@ -24,7 +24,12 @@ int main(int argc, char *argv[]) { std::string net = "dla34_cnet3d_fp32.rt"; if(argc > 1) net = argv[1]; - std::string input = "../demo/yolo_test.mp4"; + #ifdef __linux__ + std::string input = "../demo/yolo_test.mp4"; + #elif _WIN32 + std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + #endif + if(argc > 2) input = argv[2]; char ntype = 'c'; @@ -33,9 +38,18 @@ int main(int argc, char *argv[]) { int n_classes = 3; if(argc > 4) n_classes = atoi(argv[4]); - bool show = false; + int n_batch = 1; if(argc > 5) - show = atoi(argv[5]); + n_batch = atoi(argv[5]); + bool show = true; + if(argc > 6) + show = atoi(argv[6]); + float conf_thresh=0.3; + if(argc > 7) + conf_thresh = atof(argv[7]); + + if(n_batch < 1 || n_batch > 64) + FatalError("Batch dim not supported"); if(!show) SAVE_RESULT = true; @@ -57,7 +71,7 @@ int main(int argc, char *argv[]) { FatalError("Network type not allowed (3rd parameter)\n"); } - detNN->init(net, n_classes); + detNN->init(net, n_classes, n_batch, conf_thresh); gRun = true; @@ -75,30 +89,40 @@ int main(int argc, char *argv[]) { } cv::Mat frame; - cv::Mat dnn_input; if(show) - cv::namedWindow("detection", cv::WINDOW_NORMAL); + cv::namedWindow("detection", cv::WINDOW_NORMAL); - std::vector detected_bbox; + std::vector batch_frame; + std::vector batch_dnn_input; while(gRun) { - cap >> frame; - if(!frame.data) { - break; - } - - // this will be resized to the net format - dnn_input = frame.clone(); + batch_dnn_input.clear(); + batch_frame.clear(); + for(int bi=0; bi< n_batch; ++bi){ + cap >> frame; + if(!frame.data) + break; + + batch_frame.push_back(frame); + + // this will be resized to the net format + batch_dnn_input.push_back(frame.clone()); + } + if(!frame.data) + break; + //inference - detNN->update(dnn_input); - frame = detNN->draw(frame); - - if(show) { - cv::imshow("detection", frame); - cv::waitKey(1); - } - if(SAVE_RESULT) + detNN->update(batch_dnn_input, n_batch); + detNN->draw(batch_frame); + + if(show){ + for(int bi=0; bi< n_batch; ++bi){ + cv::imshow("detection", batch_frame[bi]); + cv::waitKey(1); + } + } + if(n_batch == 1 && SAVE_RESULT) resultVideo << frame; } @@ -124,7 +148,6 @@ int main(int argc, char *argv[]) { std::cout<<"Avg: "< detected3D; - std::vectorcls3D; std::vector> face_id; public: CenternetDetection3D() {}; ~CenternetDetection3D() {}; - bool init(const std::string& tensor_path, const int n_classes=3); - void preprocess(cv::Mat &frame); - void postprocess(); - cv::Mat draw(cv::Mat &frame); + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); + void preprocess(cv::Mat &frame, const int bi=0); + void postprocess(const int bi=0,const bool mAP=false); + void draw(std::vector& frames); }; diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 5149d70..809d0c9 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -145,6 +145,7 @@ private: int count_det; //tracks std::vector tr_res; + std::vector> batchTracked; int count_tr; int track_id=0; @@ -153,7 +154,7 @@ private: bool init_pre_inf(); bool init_postprocessing(); bool init_visualization(const int n_classes); - void pre_inf(); + void pre_inf(const int bi); void _get_additional_inputs(); cv::Mat transform_preds_with_trans(float x1, float x2); void tracking(); @@ -161,11 +162,11 @@ private: public: tk::dnn::Network *pre_phase_net = nullptr; CenternetDetection3DTrack() {}; - ~CenternetDetection3DTrack() {}; - bool init(const std::string& tensor_path, const int n_classes=3); - void preprocess(cv::Mat &frame); - void postprocess(); - cv::Mat draw(cv::Mat &frame); + ~CenternetDetection3DTrack() {}; + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); + void preprocess(cv::Mat &frame, const int bi=0); + void postprocess(const int bi=0,const bool mAP=false); + void draw(std::vector& frames); }; diff --git a/include/tkDNN/DetectionNN3D.h b/include/tkDNN/DetectionNN3D.h index 2870aa0..65cd728 100644 --- a/include/tkDNN/DetectionNN3D.h +++ b/include/tkDNN/DetectionNN3D.h @@ -3,8 +3,11 @@ #include #include -#include +#include +#ifdef __linux__ #include +#endif + #include #include "utils.h" @@ -30,10 +33,12 @@ class DetectionNN3D { tk::dnn::NetworkRT *netRT = nullptr; dnnType *input_d; - cv::Size originalSize; + std::vector originalSize; cv::Scalar colors[256]; + int nBatches = 1; + #ifdef OPENCV_CUDACONTRIB cv::cuda::GpuMat bgr[3]; cv::cuda::GpuMat imagePreproc; @@ -47,21 +52,26 @@ class DetectionNN3D { * This method preprocess the image, before feeding it to the NN. * * @param frame original frame to adapt for inference. + * @param bi batch index */ - virtual void preprocess(cv::Mat &frame) = 0; + virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; /** * This method postprocess the output of the NN to obtain the correct * boundig boxes. * + * @param bi batch index + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation */ - virtual void postprocess() = 0; + virtual void postprocess(const int bi=0,const bool mAP=false) = 0; public: int classes = 0; float confThreshold = 0.3; /*threshold on the confidence of the boxes*/ - - std::vector detected; /*bounding boxes in output*/ + + std::vector detected3D; /*bounding boxes in output*/ + std::vector> batchDetected; /*bounding boxes in output*/ std::vector pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/ std::vector classesNames; @@ -69,68 +79,79 @@ class DetectionNN3D { ~DetectionNN3D(){}; /** - * Method used to inialize the class, allocate memory and compute + * Method used to initialize the class, allocate memory and compute * needed data. * - * @param tensor_path path to the rt file og the NN. + * @param tensor_path path to the rt file of the NN. * @param n_classes number of classes for the given dataset. + * @param n_batches maximum number of batches to use in inference. * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path, const int n_classes=3) = 0; - - /** - * Method to draw boundixg boxes and labels on a frame. - * - * @param frame orginal frame to draw bounding box on. - * @return frame with boundig boxes. - */ - virtual cv::Mat draw(cv::Mat &frame){}; + virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3) = 0; /** * This method performs the whole detection of the NN. * - * @param frame frame to run detection on. + * @param frames frames to run detection on. + * @param cur_batches number of batches to use in inference. * @param save_times if set to true, preprocess, inference and postprocess times * are saved on a csv file, otherwise not. - * @param times pointer to the output stream where to write times + * @param times pointer to the output stream where to write times. + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation. */ - void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){ - if(!frame.data) - FatalError("No image data feed to detection"); - + void update(std::vector& frames, const int cur_batches=1, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ if(save_times && times==nullptr) FatalError("save_times set to true, but no valid ofstream given"); + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); - originalSize = frame.size(); - printCenteredTitle(" TENSORRT detection ", '=', 30); + originalSize.clear(); + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); { TKDNN_TSTART - preprocess(frame); + for(int bi=0; biinput_dim; + dim.n = cur_batches; { - dim.print(); + if(TKDNN_VERBOSE) dim.print(); TKDNN_TSTART netRT->infer(dim, input_d); TKDNN_TSTOP - dim.print(); + if(TKDNN_VERBOSE) dim.print(); stats.push_back(t_ns); if(save_times) *times<& frames){}; + }; }} diff --git a/src/CenternetDetection3D.cpp b/src/CenternetDetection3D.cpp index 1803381..53b3cf7 100644 --- a/src/CenternetDetection3D.cpp +++ b/src/CenternetDetection3D.cpp @@ -3,10 +3,12 @@ namespace tk { namespace dnn { -bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes){ +bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; + nBatches = n_batches; + confThreshold = conf_thresh; dim = netRT->input_dim; @@ -28,7 +30,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas trans = cv::Mat(cv::Size(3,2), CV_32F); trans2 = cv::Mat(cv::Size(3,2), CV_32F); - checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); dim_hm = tk::dnn::dataDim_t(1, 3, 128, 128, 1); dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); @@ -91,7 +93,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); #else - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); mean << 0.485, 0.456, 0.406; stddev << 0.229, 0.224, 0.225; #endif @@ -154,13 +156,13 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } -void CenternetDetection3D::preprocess(cv::Mat &frame){ +void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){ // -----------------------------------pre-process ------------------------------------------ // auto start_t = std::chrono::steady_clock::now(); // auto step_t = std::chrono::steady_clock::now(); // auto end_t = std::chrono::steady_clock::now(); - cv::Size sz = originalSize; + cv::Size sz = originalSize[bi]; // std::cout<<"image: "<(end_t - step_t).count() << " us" << std::endl; // step_t = end_t; - checkCuda(cudaMemcpy(input_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); // end_t = std::chrono::steady_clock::now(); // std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; @@ -280,21 +282,21 @@ void CenternetDetection3D::preprocess(cv::Mat &frame){ int idx = i*imageF.rows*imageF.cols; int ch = dim2.c-3 +i; // std::cout<<"i: "<input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType)); } - checkCuda(cudaMemcpyAsync(input_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice)); + checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice)); #endif } -void CenternetDetection3D::postprocess(){ +void CenternetDetection3D::postprocess(const int bi, const bool mAP) { dnnType *rt_out[7]; - rt_out[0] = (dnnType *)netRT->buffersRT[1]; - rt_out[1] = (dnnType *)netRT->buffersRT[2]; - rt_out[2] = (dnnType *)netRT->buffersRT[3]; - rt_out[3] = (dnnType *)netRT->buffersRT[4]; - rt_out[4] = (dnnType *)netRT->buffersRT[5]; - rt_out[5] = (dnnType *)netRT->buffersRT[6]; - rt_out[6] = (dnnType *)netRT->buffersRT[7]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi; + rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi; + rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi; + rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi; + rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi; + rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi; // ------------------------------------ process -------------------------------------------- activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot()); @@ -404,8 +406,7 @@ void CenternetDetection3D::postprocess(){ if(rot_y peakThreshold) { - if(scores[j] > centerThreshold) { + if(scores[j] > confThreshold) { if(z>0) { // compute_box_3d r.at(0,0) = std::cos(rot_y); @@ -457,16 +458,17 @@ void CenternetDetection3D::postprocess(){ } res.cl = i; res.prob = scores[j]; - res.print(); + //res.print(); detected3D.push_back(res); } } } } } + batchDetected.push_back(detected3D); } -cv::Mat CenternetDetection3D::draw(cv::Mat &frame) { +void CenternetDetection3D::draw(std::vector& frames) { tk::dnn::box3D b; int x0, w, x1, y0, h, y1; int objClass; @@ -476,40 +478,41 @@ cv::Mat CenternetDetection3D::draw(cv::Mat &frame) { float font_scale = 0.5; int thickness = 2; - // draw dets - for(int i=0; i=0; ind_f--) { - for(int j=0; j<4; j++) { - cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), - b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), - colors[b.cl], 2); - if(ind_f == 0) { - cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), - b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), - b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); - cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), - b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), - b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), + b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + colors[b.cl], 2); + if(ind_f == 0) { + cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), + b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), + b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); + cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), + b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), + b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + } } } + // draw label + cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); + cv::rectangle(frames[bi], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1)), + cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), + (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); + cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); } - // draw label - cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); - cv::rectangle(frame, cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1)), - cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), - (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); - cv::putText(frame, classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), - cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); } - return frame; } }} diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index ef3e161..dfc38f9 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -3,12 +3,14 @@ namespace tk { namespace dnn { -bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes){ - std::cout<<(tensor_path).c_str()<<"\n"; + +bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); dim = netRT->input_dim; dim.c = 3; + nBatches = n_batches; + confThreshold = conf_thresh; init_preprocessing(); init_pre_inf(); @@ -46,13 +48,13 @@ bool CenternetDetection3DTrack::init_preprocessing(){ checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); #else - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot())); + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot() * nBatches)); mean << 0.40789655, 0.44719303, 0.47026116; stddev << 0.2886383, 0.27408165, 0.27809834; #endif - checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); checkCuda(cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); } @@ -276,20 +278,20 @@ void CenternetDetection3DTrack::_get_additional_inputs(){ //None no additional input } -void CenternetDetection3DTrack::pre_inf(){ +void CenternetDetection3DTrack::pre_inf(const int bi){ TKDNN_TSTART tk::dnn::dataDim_t dim_aus; pre_phase_net->infer(dim_aus, nullptr); TKDNN_TSTOP checkCuda( cudaDeviceSynchronize() ); - checkCuda( cudaMemcpy(input_d, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); checkCuda( cudaDeviceSynchronize() ); } -void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ - // -----------------------------------pre-process ------------------------------------------ - - cv::Size sz = originalSize; +void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ + // -----------------------------------pre-process ------------------------------------------ + batchTracked.clear(); + cv::Size sz = originalSize[bi]; cv::Size sz_old; float scale = 1.0; float new_height = sz.height * scale; @@ -302,7 +304,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ // float s = new_width >= new_height ? new_width : new_height; // ----------- get_affine_transform // rot_rad = pi * 0 / 100 --> 0 - dim.print(); + //dim.print(); src.at(0,0)=c[0]; src.at(0,1)=c[1]; src.at(1,0)=c[0]; @@ -389,7 +391,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ checkCuda( cudaDeviceSynchronize() ); iter0=false; } - pre_inf(); + pre_inf(bi); checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); checkCuda( cudaDeviceSynchronize() ); @@ -587,17 +589,17 @@ void CenternetDetection3DTrack::tracking(){ } -void CenternetDetection3DTrack::postprocess(){ +void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { dnnType *rt_out[9]; - rt_out[0] = (dnnType *)netRT->buffersRT[1]; - rt_out[1] = (dnnType *)netRT->buffersRT[2]; - rt_out[2] = (dnnType *)netRT->buffersRT[3]; - rt_out[3] = (dnnType *)netRT->buffersRT[4]; - rt_out[4] = (dnnType *)netRT->buffersRT[5]; - rt_out[5] = (dnnType *)netRT->buffersRT[6]; - rt_out[6] = (dnnType *)netRT->buffersRT[7]; - rt_out[7] = (dnnType *)netRT->buffersRT[8]; - rt_out[8] = (dnnType *)netRT->buffersRT[9]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi; + rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi; + rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi; + rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi; + rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi; + rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi; + rt_out[7] = (dnnType *)netRT->buffersRT[8]+ netRT->buffersDIM[8].tot()*bi; + rt_out[8] = (dnnType *)netRT->buffersRT[9]+ netRT->buffersDIM[9].tot()*bi; // ------------------------------------ process -------------------------------------------- @@ -719,143 +721,144 @@ void CenternetDetection3DTrack::postprocess(){ } // track step tracking(); + batchTracked.push_back(tr_res); } -cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) { - +void CenternetDetection3DTrack::draw(std::vector& frames) { + struct trackingRes t; float sc; int id; std::string txt; int baseline = 0; float font_scale = 0.8; - int thickness = 2; - for(int i=0; i vis_thresh){// && tr_res[i].active!=0) { - if(view2d) { - - - cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), tr_res[i].det_res.bb0.at(0,1)), - cv::Point(tr_res[i].det_res.bb1.at(0,0), tr_res[i].det_res.bb1.at(0,1)), tr_colors[tr_res[i].color], thickness); - cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), - tr_res[i].det_res.bb0.at(0,1) - text_size.height - thickness), - cv::Point(tr_res[i].det_res.bb0.at(0,0) + text_size.width, - tr_res[i].det_res.bb0.at(0,1)), tr_colors[tr_res[i].color], -1); - - cv::putText(frame, txt, cv::Point(tr_res[i].det_res.bb0.at(0,0), - tr_res[i].det_res.bb0.at(0,1) - thickness -1), - cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); + int thickness = 2; + for(int bi=0; bi vis_thresh){// && t.active!=0) { + if(view2d) { + cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), t.det_res.bb0.at(0,1)), + cv::Point(t.det_res.bb1.at(0,0), t.det_res.bb1.at(0,1)), tr_colors[t.color], thickness); + cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), + t.det_res.bb0.at(0,1) - text_size.height - thickness), + cv::Point(t.det_res.bb0.at(0,0) + text_size.width, + t.det_res.bb0.at(0,1)), tr_colors[t.color], -1); + + cv::putText(frames[bi], txt, cv::Point(t.det_res.bb0.at(0,0), + t.det_res.bb0.at(0,1) - thickness -1), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frame, cv::Point((int)tr_res[i].det_res.ct.at(0,0), - (int)tr_res[i].det_res.ct.at(0,1)), - cv::Point((int)(tr_res[i].det_res.ct.at(0,0) + tr_res[i].det_res.tr.at(0,0)), - (int)(tr_res[i].det_res.ct.at(0,1) + tr_res[i].det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); - } - //3d - if(!view2d && tr_res[i].det_res.z > 1){ - r.at(0,0) = std::cos(tr_res[i].det_res.rot_y); - r.at(0,2) = std::sin(tr_res[i].det_res.rot_y); - r.at(2,0) = -std::sin(tr_res[i].det_res.rot_y); - r.at(2,2) = std::cos(tr_res[i].det_res.rot_y); - - corners.at(0,0) = tr_res[i].det_res.dim[2]/2; - corners.at(0,1) = tr_res[i].det_res.dim[2]/2; - corners.at(0,2) = -tr_res[i].det_res.dim[2]/2; - corners.at(0,3) = -tr_res[i].det_res.dim[2]/2; - corners.at(0,4) = tr_res[i].det_res.dim[2]/2; - corners.at(0,5) = tr_res[i].det_res.dim[2]/2; - corners.at(0,6) = -tr_res[i].det_res.dim[2]/2; - corners.at(0,7) = -tr_res[i].det_res.dim[2]/2; - - corners.at(1,4) = -tr_res[i].det_res.dim[0]; - corners.at(1,5) = -tr_res[i].det_res.dim[0]; - corners.at(1,6) = -tr_res[i].det_res.dim[0]; - corners.at(1,7) = -tr_res[i].det_res.dim[0]; - - corners.at(2,0) = tr_res[i].det_res.dim[1]/2; - corners.at(2,1) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,2) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,3) = tr_res[i].det_res.dim[1]/2; - corners.at(2,4) = tr_res[i].det_res.dim[1]/2; - corners.at(2,5) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,6) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,7) = tr_res[i].det_res.dim[1]/2; - - cv::Mat aus = r * corners; - - for(int k=0; k<8; k++) { - aus.at(0,k) += tr_res[i].det_res.x; - aus.at(1,k) += tr_res[i].det_res.y; - aus.at(2,k) += tr_res[i].det_res.z; + cv::arrowedLine(frames[bi], cv::Point((int)t.det_res.ct.at(0,0), + (int)t.det_res.ct.at(0,1)), + cv::Point((int)(t.det_res.ct.at(0,0) + t.det_res.tr.at(0,0)), + (int)(t.det_res.ct.at(0,1) + t.det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); } - - // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); - for(int k1=0; k1<3; k1++) { - for(int k2=0; k2<8; k2++) - pts3DHomo.at(k1,k2) = aus.at(k1,k2); - } - - aus.release(); - aus = calibs * pts3DHomo; - std::vector res_corners; - for(int k=0; k<8; k++) { - res_corners.push_back(aus.at(0,k) / aus.at(2,k)); - res_corners.push_back(aus.at(1,k) / aus.at(2,k)); - } - aus.release(); - for(int ind_f = 3; ind_f>=0; ind_f--) { - for(int j=0; j<4; j++) { - cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), - (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), - tr_colors[tr_res[i].color], 2); - if(ind_f == 0 && j==3) { - cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), - (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), - (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2); - cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), - (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), - (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2); + //3d + if(!view2d && t.det_res.z > 1){ + r.at(0,0) = std::cos(t.det_res.rot_y); + r.at(0,2) = std::sin(t.det_res.rot_y); + r.at(2,0) = -std::sin(t.det_res.rot_y); + r.at(2,2) = std::cos(t.det_res.rot_y); + + corners.at(0,0) = t.det_res.dim[2]/2; + corners.at(0,1) = t.det_res.dim[2]/2; + corners.at(0,2) = -t.det_res.dim[2]/2; + corners.at(0,3) = -t.det_res.dim[2]/2; + corners.at(0,4) = t.det_res.dim[2]/2; + corners.at(0,5) = t.det_res.dim[2]/2; + corners.at(0,6) = -t.det_res.dim[2]/2; + corners.at(0,7) = -t.det_res.dim[2]/2; + + corners.at(1,4) = -t.det_res.dim[0]; + corners.at(1,5) = -t.det_res.dim[0]; + corners.at(1,6) = -t.det_res.dim[0]; + corners.at(1,7) = -t.det_res.dim[0]; + + corners.at(2,0) = t.det_res.dim[1]/2; + corners.at(2,1) = -t.det_res.dim[1]/2; + corners.at(2,2) = -t.det_res.dim[1]/2; + corners.at(2,3) = t.det_res.dim[1]/2; + corners.at(2,4) = t.det_res.dim[1]/2; + corners.at(2,5) = -t.det_res.dim[1]/2; + corners.at(2,6) = -t.det_res.dim[1]/2; + corners.at(2,7) = t.det_res.dim[1]/2; + + cv::Mat aus = r * corners; + + for(int k=0; k<8; k++) { + aus.at(0,k) += t.det_res.x; + aus.at(1,k) += t.det_res.y; + aus.at(2,k) += t.det_res.z; + } + + // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); + for(int k1=0; k1<3; k1++) { + for(int k2=0; k2<8; k2++) + pts3DHomo.at(k1,k2) = aus.at(k1,k2); + } + + aus.release(); + aus = calibs * pts3DHomo; + std::vector res_corners; + for(int k=0; k<8; k++) { + res_corners.push_back(aus.at(0,k) / aus.at(2,k)); + res_corners.push_back(aus.at(1,k) / aus.at(2,k)); + } + aus.release(); + for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), + (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + tr_colors[t.color], 2); + if(ind_f == 0 && j==3) { + cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), + (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), + (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[t.color], 2); + cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), + (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), + (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[t.color], 2); + } } } - } - float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; - for(int k=0; k<8; k++) { - if(res_corners[2*k]bb1) - bb1=res_corners[2*k]; - if(res_corners[2*k+1]bb3) - bb3=res_corners[2*k+1]; - - } - // if(not no_bbox): - // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), - // tr_colors[tr_res[i].color], thickness); - cv::rectangle(frame, cv::Point(bb0, bb2 - text_size.height - thickness), - cv::Point(bb0 + text_size.width, bb2), tr_colors[tr_res[i].color], -1); - - cv::putText(frame, txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, - font_scale, cv::Scalar(255, 255, 255), 1); + float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; + for(int k=0; k<8; k++) { + if(res_corners[2*k]bb1) + bb1=res_corners[2*k]; + if(res_corners[2*k+1]bb3) + bb3=res_corners[2*k+1]; + + } + // if(not no_bbox): + // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), + // tr_colors[t.color], thickness); + cv::rectangle(frames[bi], cv::Point(bb0, bb2 - text_size.height - thickness), + cv::Point(bb0 + text_size.width, bb2), tr_colors[t.color], -1); + + cv::putText(frames[bi], txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, + font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frame, cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), - cv::Point((int)((bb0 + bb1)/2 + tr_res[i].det_res.tr.at(0,0)), - (int)((bb2 + bb3)/2 + tr_res[i].det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); + cv::arrowedLine(frames[bi], cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), + cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at(0,0)), + (int)((bb2 + bb3)/2 + t.det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); + } } } - } - return frame; } }} -- 2.52.0 From 2367519799ef3eb9806387dab94ec12d17e77649 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 30 Apr 2021 17:10:51 +0200 Subject: [PATCH 126/228] Add the calibration matrix reading for CenterTrack Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 25 +++++++-- include/tkDNN/CenternetDetection3D.h | 4 +- include/tkDNN/CenternetDetection3DTrack.h | 10 ++-- include/tkDNN/DetectionNN3D.h | 10 ++-- src/CenternetDetection3D.cpp | 5 +- src/CenternetDetection3DTrack.cpp | 65 +++++++++++++---------- 6 files changed, 75 insertions(+), 44 deletions(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index c35ac51..558e6af 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -70,8 +70,17 @@ int main(int argc, char *argv[]) { default: FatalError("Network type not allowed (3rd parameter)\n"); } - - detNN->init(net, n_classes, n_batch, conf_thresh); + std::vector calibs; + // cv::Mat calib = cv::Mat::zeros(cv::Size(3,3), CV_32F); + // calib.at(0,0) = 864.1243196486207;// * 512.0;//884.081444212;//864.1243196486207 * 512.0;// 633.0; + // calib.at(0,2) = 726.7271690557819;// * 512.0;//0.0;//726.7271690557819 * 512.0;// 0.0; //w/2 + // calib.at(1,1) = 883.6552349216504;// * 512.0;//884.081444212;//883.6552349216504 * 512.0;// 633.0; + // calib.at(1,2) = 506.8548506986564;// * 512.0;//0.0;//506.8548506986564 * 512.0;// 0.0; //h/2 + // calibs.push_back(calib); + // calibs.push_back(calib); + // calibs.push_back(calib); + // calibs.push_back(calib); + detNN->init(net, n_classes, n_batch, conf_thresh, calibs); gRun = true; @@ -87,7 +96,8 @@ int main(int argc, char *argv[]) { int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); } - + cv::Size sz_resize = cv::Size(512,512); + std::vector sz_orig; cv::Mat frame; if(show) cv::namedWindow("detection", cv::WINDOW_NORMAL); @@ -98,12 +108,15 @@ int main(int argc, char *argv[]) { while(gRun) { batch_dnn_input.clear(); batch_frame.clear(); + sz_orig.clear(); for(int bi=0; bi< n_batch; ++bi){ cap >> frame; if(!frame.data) break; - + sz_orig.push_back(frame.size()); + if(calibs.size() != 0) + resize(frame, frame, sz_resize); batch_frame.push_back(frame); // this will be resized to the net format @@ -113,11 +126,13 @@ int main(int argc, char *argv[]) { break; //inference - detNN->update(batch_dnn_input, n_batch); + detNN->update(batch_dnn_input, n_batch, false, nullptr, false, sz_orig); detNN->draw(batch_frame); if(show){ for(int bi=0; bi< n_batch; ++bi){ + if(calibs.size() != 0) + resize(batch_frame[bi], batch_frame[bi], sz_orig[bi]); cv::imshow("detection", batch_frame[bi]); cv::waitKey(1); } diff --git a/include/tkDNN/CenternetDetection3D.h b/include/tkDNN/CenternetDetection3D.h index 668440c..cbffa22 100644 --- a/include/tkDNN/CenternetDetection3D.h +++ b/include/tkDNN/CenternetDetection3D.h @@ -82,8 +82,8 @@ public: CenternetDetection3D() {}; ~CenternetDetection3D() {}; - bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); - void preprocess(cv::Mat &frame, const int bi=0); + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); + void preprocess(cv::Mat &frame, const int bi=0, const std::vector& stream_size=std::vector()); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); }; diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 809d0c9..451bf6e 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -74,6 +74,10 @@ private: #endif float *d_ptrs; + std::vector inputCalibs; + + std::vector sz_old; + cv::Mat src; cv::Mat dst; cv::Mat dst2; @@ -124,7 +128,7 @@ private: /* visualization */ cv::Mat r; - cv::Mat calibs; + std::vector calibs; cv::Mat corners, pts3DHomo; std::vector> face_id; @@ -163,8 +167,8 @@ public: tk::dnn::Network *pre_phase_net = nullptr; CenternetDetection3DTrack() {}; ~CenternetDetection3DTrack() {}; - bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); - void preprocess(cv::Mat &frame, const int bi=0); + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); + void preprocess(cv::Mat &frame, const int bi=0, const std::vector& stream_size=std::vector()); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); }; diff --git a/include/tkDNN/DetectionNN3D.h b/include/tkDNN/DetectionNN3D.h index 65cd728..7320bef 100644 --- a/include/tkDNN/DetectionNN3D.h +++ b/include/tkDNN/DetectionNN3D.h @@ -54,7 +54,7 @@ class DetectionNN3D { * @param frame original frame to adapt for inference. * @param bi batch index */ - virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; + virtual void preprocess(cv::Mat &frame, const int bi=0 , const std::vector& stream_size=std::vector()) = 0; /** * This method postprocess the output of the NN to obtain the correct @@ -87,7 +87,8 @@ class DetectionNN3D { * @param n_batches maximum number of batches to use in inference. * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3) = 0; + virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, + const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()) = 0; /** * This method performs the whole detection of the NN. @@ -100,7 +101,8 @@ class DetectionNN3D { * @param mAP set to true only if all the probabilities for a bounding * box are needed, as in some cases for the mAP calculation. */ - void update(std::vector& frames, const int cur_batches=1, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ + void update(std::vector& frames, const int cur_batches=1, bool save_times=false, + std::ofstream *times=nullptr, const bool mAP=false, const std::vector& stream_size=std::vector()){ if(save_times && times==nullptr) FatalError("save_times set to true, but no valid ofstream given"); if(cur_batches > nBatches) @@ -114,7 +116,7 @@ class DetectionNN3D { if(!frames[bi].data) FatalError("No image data feed to detection"); originalSize.push_back(frames[bi].size()); - preprocess(frames[bi], bi); + preprocess(frames[bi], bi, stream_size); } TKDNN_TSTOP pre_stats.push_back(t_ns); diff --git a/src/CenternetDetection3D.cpp b/src/CenternetDetection3D.cpp index 53b3cf7..73e4215 100644 --- a/src/CenternetDetection3D.cpp +++ b/src/CenternetDetection3D.cpp @@ -3,7 +3,8 @@ namespace tk { namespace dnn { -bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { +bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches, + const float conf_thresh, const std::vector& k_calibs) { std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; @@ -156,7 +157,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } -void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){ +void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi, const std::vector& stream_size){ // -----------------------------------pre-process ------------------------------------------ // auto start_t = std::chrono::steady_clock::now(); diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index dfc38f9..02db674 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -4,14 +4,15 @@ namespace tk { namespace dnn { -bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { +bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, + const float conf_thresh, const std::vector& k_calibs) { netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); dim = netRT->input_dim; dim.c = 3; nBatches = n_batches; confThreshold = conf_thresh; - + inputCalibs = k_calibs; init_preprocessing(); init_pre_inf(); init_postprocessing(); @@ -37,7 +38,10 @@ bool CenternetDetection3DTrack::init_preprocessing(){ dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); - + for(int bi=0; bi(0,0) = 633.0; - calibs.at(0,1) = 0.0; - calibs.at(0,2) = 0.0; //w/2 - calibs.at(0,3) = 0.0; - calibs.at(1,0) = 0.0; - calibs.at(1,1) = 633.0; - calibs.at(1,2) = 0.0; //h/2 - calibs.at(1,3) = 0.0; - calibs.at(2,0) = 0.0; - calibs.at(2,1) = 0.0; - calibs.at(2,2) = 1.0; - calibs.at(2,3) = 0.0; + for(int bi=0; bi(0,0) = 633.0; + calibs_.at(1,1) = 633.0; + calibs_.at(2,2) = 1.0; + } + calibs_.at(2,2) = 1.0; + calibs.push_back(calibs_); + } // Alloc array used in the kernel checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); @@ -288,17 +289,25 @@ void CenternetDetection3DTrack::pre_inf(const int bi){ checkCuda( cudaDeviceSynchronize() ); } -void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ +void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const std::vector& stream_size){ // -----------------------------------pre-process ------------------------------------------ batchTracked.clear(); cv::Size sz = originalSize[bi]; - cv::Size sz_old; float scale = 1.0; float new_height = sz.height * scale; float new_width = sz.width * scale; - if(sz.height != sz_old.height && sz.width != sz_old.width){ - calibs.at(0,2) = new_width / 2.0f; - calibs.at(1,2) = new_height /2.0f; + if(sz.height != sz_old[bi].height && sz.width != sz_old[bi].width){ + if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) { + calibs[bi].at(0,2) = new_width / 2.0f; + calibs[bi].at(1,2) = new_height /2.0f; + } + else { + calibs[bi].at(0,0) = inputCalibs[bi].at(0,0) * dim.w / stream_size[bi].width; + calibs[bi].at(0,2) = inputCalibs[bi].at(0,2) * dim.w / stream_size[bi].width; + calibs[bi].at(1,1) = inputCalibs[bi].at(1,1) * dim.h / stream_size[bi].height; + calibs[bi].at(1,2) = inputCalibs[bi].at(1,2) * dim.h / stream_size[bi].height; + } + float c[] = {new_width / 2.0f, new_height /2.0f}; float s[] = {dim.w, dim.h}; // float s = new_width >= new_height ? new_width : new_height; @@ -324,7 +333,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ trans2 = cv::getAffineTransform( dst2, src ); trans2.convertTo(trans_out, CV_32F); } - sz_old = sz; + sz_old[bi] = sz; #ifdef OPENCV_CUDACONTRIB std::cout<<"OPENCV CPMTROB\n"; cv::cuda::GpuMat im_Orig; @@ -358,7 +367,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ #else std::cout<<"NO OPENCV CPMTROB\n"; cv::Mat imageF; - // resize(frame, imageF, cv::Size(new_width, new_height)); + //resize(frame, imageF, cv::Size(512, 512)); imageF = frame; sz = imageF.size(); @@ -701,9 +710,9 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { new_det_res.dim[2] = dim_[i+2*K]; // unproject_2d_to_3d - new_det_res.z = dep[i] - calibs.at(2,3); - new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs.at(0,3) - calibs.at(0,2) * new_det_res.z) / calibs.at(0,0); - new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs.at(1,3) - calibs.at(1,2) * new_det_res.z) / calibs.at(1,1) + (dim_[i] / 2); + new_det_res.z = dep[i] - calibs[bi].at(2,3); + new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs[bi].at(0,3) - calibs[bi].at(0,2) * new_det_res.z) / calibs[bi].at(0,0); + new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs[bi].at(1,3) - calibs[bi].at(1,2) * new_det_res.z) / calibs[bi].at(1,1) + (dim_[i] / 2); // alpha2rot_y // idx = rot[:, 1] > rot[:, 5] @@ -714,7 +723,7 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { new_det_res.alpha = std::atan2(rot[2*K + i], rot[3*K + i]) -0.5 * M_PI; else new_det_res.alpha = std::atan2(rot[6*K + i], rot[7*K + i]) +0.5 * M_PI; - new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at(0,0) - calibs.at(0,2), calibs.at(0,0))); + new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at(0,0) - calibs[bi].at(0,2), calibs[bi].at(0,0))); new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest det_res.push_back(new_det_res); @@ -804,7 +813,7 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { } aus.release(); - aus = calibs * pts3DHomo; + aus = calibs[bi] * pts3DHomo; std::vector res_corners; for(int k=0; k<8; k++) { res_corners.push_back(aus.at(0,k) / aus.at(2,k)); -- 2.52.0 From ff6e0e010adc120bd010c605f82355111758dd08 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 30 Apr 2021 22:26:33 +0200 Subject: [PATCH 127/228] Fix a bug with batch > 1 Signed-off-by: Davide Sapienza --- include/tkDNN/CenternetDetection3DTrack.h | 9 +- src/CenternetDetection3DTrack.cpp | 184 +++++++++++----------- 2 files changed, 99 insertions(+), 94 deletions(-) diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 451bf6e..258fe9d 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -148,10 +148,9 @@ private: std::vector det_res; int count_det; //tracks - std::vector tr_res; - std::vector> batchTracked; - int count_tr; - int track_id=0; + std::vector> tr_res; + std::vector count_tr; + std::vector track_id; bool init_preprocessing(); @@ -161,7 +160,7 @@ private: void pre_inf(const int bi); void _get_additional_inputs(); cv::Mat transform_preds_with_trans(float x1, float x2); - void tracking(); + void tracking(const int bi); public: tk::dnn::Network *pre_phase_net = nullptr; diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 02db674..bdc1c59 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -17,8 +17,6 @@ bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n init_pre_inf(); init_postprocessing(); init_visualization(n_classes); - - count_tr = 0; } bool CenternetDetection3DTrack::init_preprocessing(){ @@ -201,6 +199,11 @@ bool CenternetDetection3DTrack::init_postprocessing(){ // Alloc array used in the kernel checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); + + for(int bi=0; bi& stream_size){ // -----------------------------------pre-process ------------------------------------------ - batchTracked.clear(); cv::Size sz = originalSize[bi]; float scale = 1.0; float new_height = sz.height * scale; @@ -414,8 +416,7 @@ cv::Mat CenternetDetection3DTrack::transform_preds_with_trans(float x1, float x2 return trans_out * target_coords; } -void CenternetDetection3DTrack::tracking(){ - +void CenternetDetection3DTrack::tracking(const int bi) { float item_size[count_det]; int item_cl[count_det]; float dets[2*count_det]; @@ -427,44 +428,44 @@ void CenternetDetection3DTrack::tracking(){ dets[i*2+1] = det_res[i].ct.at(0,1); } - float track_size[count_tr]; - int track_cl[count_tr]; - float tracks[2*count_tr]; - for(int i=0; i(0,0) - tr_res[i].det_res.bb0.at(0,0)) * - (tr_res[i].det_res.bb1.at(0,1) - tr_res[i].det_res.bb0.at(0,1)); - track_cl[i] = tr_res[i].det_res.cl; - tracks[i*2] = tr_res[i].det_res.ct.at(0,0); - tracks[i*2+1] = tr_res[i].det_res.ct.at(0,1); + float track_size[count_tr[bi]]; + int track_cl[count_tr[bi]]; + float tracks[2*count_tr[bi]]; + for(int i=0; i(0,0) - tr_res[bi][i].det_res.bb0.at(0,0)) * + (tr_res[bi][i].det_res.bb1.at(0,1) - tr_res[bi][i].det_res.bb0.at(0,1)); + track_cl[i] = tr_res[bi][i].det_res.cl; + tracks[i*2] = tr_res[bi][i].det_res.ct.at(0,0); + tracks[i*2+1] = tr_res[bi][i].det_res.ct.at(0,1); } - float dist[count_tr*count_det]; + float dist[count_tr[bi]*count_det]; bool invalid; - for(int i=0; i track_size[i] || dist[j*count_tr+i] > item_size[j] || item_cl[j] != track_cl[i]; - dist[j*count_tr+i] = dist[j*count_tr+i] + invalid * (1 << 18); + invalid = dist[j*count_tr[bi]+i] > track_size[i] || dist[j*count_tr[bi]+i] > item_size[j] || item_cl[j] != track_cl[i]; + dist[j*count_tr[bi]+i] = dist[j*count_tr[bi]+i] + invalid * (1 << 18); } } - int matched_indices[2*count_tr]; + int matched_indices[2*count_tr[bi]]; float min_tr; int min_idtr=-1; - for(int i=0; i new_tr_res; int id_new_tr=0; - for(int i=0; i new_thresh) { count_tr_ ++; @@ -583,17 +583,24 @@ void CenternetDetection3DTrack::tracking(){ new_tr_res_.det_res.y = det_res[i].y; new_tr_res_.det_res.z = det_res[i].z; new_tr_res_.det_res.rot_y = det_res[i].rot_y; - new_tr_res_.tracking_id = track_id++; + new_tr_res_.tracking_id = track_id[bi]++; new_tr_res_.age = 1; new_tr_res_.active = 1; new_tr_res_.color = rand() % 256; - tr_res.push_back(new_tr_res_); + if(tr_res.size() <= bi) { + std::vector v_new_tr_res_; + v_new_tr_res_.push_back(new_tr_res_); + tr_res.push_back(v_new_tr_res_); + } + else + tr_res[bi].push_back(new_tr_res_); } } - count_tr = count_tr_; + + count_tr[bi] = count_tr_; - if(track_id==1000) - track_id=0; + if(track_id[bi]==1000) + track_id[bi]=0; det_res.clear(); } @@ -729,8 +736,7 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { } // track step - tracking(); - batchTracked.push_back(tr_res); + tracking(bi); } void CenternetDetection3DTrack::draw(std::vector& frames) { @@ -743,8 +749,8 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { int thickness = 2; for(int bi=0; bi Date: Mon, 3 May 2021 18:56:07 +0200 Subject: [PATCH 128/228] Fix a bug in the draw function of CenterTrack. Signed-off-by: Davide Sapienza --- src/CenternetDetection3DTrack.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index bdc1c59..312bcdb 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -749,7 +749,7 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { int thickness = 2; for(int bi=0; bi Date: Tue, 4 May 2021 11:21:05 +0200 Subject: [PATCH 129/228] Fix tracker for batch size > 1 --- include/tkDNN/CenternetDetection3DTrack.h | 12 +- src/CenternetDetection3DTrack.cpp | 168 +++++++++++----------- 2 files changed, 92 insertions(+), 88 deletions(-) diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 451bf6e..dd092fa 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -1,11 +1,11 @@ #ifndef CENTERNETDETECTION3DTRACK_H #define CENTERNETDETECTION3DTRACK_H +#include +#include "opencv2/opencv.hpp" #include "kernels.h" #include "utils.h" #include "tkdnn.h" -#include -#include "opencv2/opencv.hpp" #include #include #include // std::iota @@ -51,7 +51,7 @@ struct trackingRes class CenternetDetection3DTrack : public DetectionNN3D { -private: +public: tk::dnn::dataDim_t dim; tk::dnn::dataDim_t dim2; tk::dnn::dataDim_t dim_hm; @@ -148,9 +148,9 @@ private: std::vector det_res; int count_det; //tracks - std::vector tr_res; + std::vector> tr_res; std::vector> batchTracked; - int count_tr; + std::vector count_tr; int track_id=0; @@ -161,7 +161,7 @@ private: void pre_inf(const int bi); void _get_additional_inputs(); cv::Mat transform_preds_with_trans(float x1, float x2); - void tracking(); + void tracking(int bi); public: tk::dnn::Network *pre_phase_net = nullptr; diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 02db674..25c4c0b 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -13,12 +13,13 @@ bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n nBatches = n_batches; confThreshold = conf_thresh; inputCalibs = k_calibs; + tr_res.resize(nBatches); init_preprocessing(); init_pre_inf(); init_postprocessing(); init_visualization(n_classes); - count_tr = 0; + count_tr.resize(nBatches, 0); } bool CenternetDetection3DTrack::init_preprocessing(){ @@ -30,6 +31,7 @@ bool CenternetDetection3DTrack::init_preprocessing(){ trans2 = cv::Mat(cv::Size(3,2), CV_32F); trans_out = cv::Mat(cv::Size(3,2), CV_32F); + dst2.at(0,0)=width * 0.5; dst2.at(0,1)=width * 0.5; dst2.at(1,0)=width * 0.5; @@ -372,6 +374,8 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const s sz = imageF.size(); cv::warpAffine(imageF, imageF, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR ); + + cv::imshow("warp", imageF); sz = imageF.size(); imageF.convertTo(imageF, CV_32FC3, 1/255.0); @@ -414,7 +418,7 @@ cv::Mat CenternetDetection3DTrack::transform_preds_with_trans(float x1, float x2 return trans_out * target_coords; } -void CenternetDetection3DTrack::tracking(){ +void CenternetDetection3DTrack::tracking(int bi){ float item_size[count_det]; int item_cl[count_det]; @@ -427,44 +431,44 @@ void CenternetDetection3DTrack::tracking(){ dets[i*2+1] = det_res[i].ct.at(0,1); } - float track_size[count_tr]; - int track_cl[count_tr]; - float tracks[2*count_tr]; - for(int i=0; i(0,0) - tr_res[i].det_res.bb0.at(0,0)) * - (tr_res[i].det_res.bb1.at(0,1) - tr_res[i].det_res.bb0.at(0,1)); - track_cl[i] = tr_res[i].det_res.cl; - tracks[i*2] = tr_res[i].det_res.ct.at(0,0); - tracks[i*2+1] = tr_res[i].det_res.ct.at(0,1); + float track_size[count_tr[bi]]; + int track_cl[count_tr[bi]]; + float tracks[2*count_tr[bi]]; + for(int i=0; i(0,0) - tr_res[bi][i].det_res.bb0.at(0,0)) * + (tr_res[bi][i].det_res.bb1.at(0,1) - tr_res[bi][i].det_res.bb0.at(0,1)); + track_cl[i] = tr_res[bi][i].det_res.cl; + tracks[i*2] = tr_res[bi][i].det_res.ct.at(0,0); + tracks[i*2+1] = tr_res[bi][i].det_res.ct.at(0,1); } - float dist[count_tr*count_det]; + float dist[count_tr[bi]*count_det]; bool invalid; - for(int i=0; i track_size[i] || dist[j*count_tr+i] > item_size[j] || item_cl[j] != track_cl[i]; - dist[j*count_tr+i] = dist[j*count_tr+i] + invalid * (1 << 18); + invalid = dist[j*count_tr[bi]+i] > track_size[i] || dist[j*count_tr[bi]+i] > item_size[j] || item_cl[j] != track_cl[i]; + dist[j*count_tr[bi]+i] = dist[j*count_tr[bi]+i] + invalid * (1 << 18); } } - int matched_indices[2*count_tr]; + int matched_indices[2*count_tr[bi]]; float min_tr; int min_idtr=-1; - for(int i=0; i new_tr_res; int id_new_tr=0; - for(int i=0; i new_thresh) { count_tr_ ++; @@ -587,10 +591,10 @@ void CenternetDetection3DTrack::tracking(){ new_tr_res_.age = 1; new_tr_res_.active = 1; new_tr_res_.color = rand() % 256; - tr_res.push_back(new_tr_res_); + tr_res[bi].push_back(new_tr_res_); } } - count_tr = count_tr_; + count_tr[bi] = count_tr_; if(track_id==1000) track_id=0; @@ -729,8 +733,8 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { } // track step - tracking(); - batchTracked.push_back(tr_res); + tracking(bi); + batchTracked.push_back(tr_res[bi]); } void CenternetDetection3DTrack::draw(std::vector& frames) { -- 2.52.0 From 0dc96d2a9e0070f92d12588f9b9da4b931188a11 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 4 May 2021 17:59:20 +0200 Subject: [PATCH 130/228] Improve CenterTrack. Signed-off-by: Davide Sapienza --- demo/demo/demo3D.cpp | 10 +- include/tkDNN/CenternetDetection3D.h | 13 +- include/tkDNN/CenternetDetection3DTrack.h | 30 +- include/tkDNN/DetectionNN3D.h | 6 +- src/CenternetDetection3D.cpp | 107 ++-- src/CenternetDetection3DTrack.cpp | 627 +++++++++++----------- 6 files changed, 397 insertions(+), 396 deletions(-) diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 558e6af..620b0d4 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -96,8 +96,6 @@ int main(int argc, char *argv[]) { int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); } - cv::Size sz_resize = cv::Size(512,512); - std::vector sz_orig; cv::Mat frame; if(show) cv::namedWindow("detection", cv::WINDOW_NORMAL); @@ -108,15 +106,11 @@ int main(int argc, char *argv[]) { while(gRun) { batch_dnn_input.clear(); batch_frame.clear(); - sz_orig.clear(); for(int bi=0; bi< n_batch; ++bi){ cap >> frame; if(!frame.data) break; - sz_orig.push_back(frame.size()); - if(calibs.size() != 0) - resize(frame, frame, sz_resize); batch_frame.push_back(frame); // this will be resized to the net format @@ -126,13 +120,11 @@ int main(int argc, char *argv[]) { break; //inference - detNN->update(batch_dnn_input, n_batch, false, nullptr, false, sz_orig); + detNN->update(batch_dnn_input, n_batch, false, nullptr, false); detNN->draw(batch_frame); if(show){ for(int bi=0; bi< n_batch; ++bi){ - if(calibs.size() != 0) - resize(batch_frame[bi], batch_frame[bi], sz_orig[bi]); cv::imshow("detection", batch_frame[bi]); cv::waitKey(1); } diff --git a/include/tkDNN/CenternetDetection3D.h b/include/tkDNN/CenternetDetection3D.h index cbffa22..943fbf4 100644 --- a/include/tkDNN/CenternetDetection3D.h +++ b/include/tkDNN/CenternetDetection3D.h @@ -27,6 +27,8 @@ private: tk::dnn::dataDim_t dim_dep; tk::dnn::dataDim_t dim_rot; tk::dnn::dataDim_t dim_dim; + + std::vector inputCalibs; float *topk_scores; int *topk_inds_; float *topk_ys_; @@ -58,32 +60,33 @@ private: dnnType *input; #endif cv::Mat r; - cv::Mat calibs; float *d_ptrs; cv::Mat src; cv::Mat dst; cv::Mat dst2; cv::Mat trans, trans2; + std::vector calibs; + //processing int K = 100; int width = 128;//56; // TODO // pointer used in the kernels - float *src_out; - int *ids_out; + float *srcOut; + int *idsOut; struct threshold op; cv::Mat corners, pts3DHomo; - std::vector> face_id; + std::vector> faceId; public: CenternetDetection3D() {}; ~CenternetDetection3D() {}; bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); - void preprocess(cv::Mat &frame, const int bi=0, const std::vector& stream_size=std::vector()); + void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); }; diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 4c036f4..c500412 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -76,12 +76,12 @@ public: std::vector inputCalibs; - std::vector sz_old; + std::vector szOld; cv::Mat src; cv::Mat dst; cv::Mat dst2; - cv::Mat trans, trans2, trans_out; + cv::Mat trans, trans2, transOut; /* pre inf */ bool iter0; @@ -131,27 +131,25 @@ public: std::vector calibs; cv::Mat corners, pts3DHomo; - std::vector> face_id; - cv::Scalar tr_colors[256]; + std::vector> faceId; + cv::Scalar trColors[256]; bool view2d = false; //processing struct threshold op; - float out_thresh = 0.1; - float new_thresh = 0.3; - float vis_thresh = 0.3; - float peakThreshold = 0.2; - float centerThreshold = 0.3; //default 0.5 + float outThresh = 0.1; + float newThresh = 0.3; + // float peakThreshold = 0.2; + // float centerThreshold = 0.3; //default 0.5 //detections - std::vector det_res; - int count_det; + std::vector detRes; + int countDet; //tracks - std::vector> tr_res; - std::vector> batchTracked; - std::vector count_tr; - std::vector track_id; + std::vector> trRes; + std::vector countTr; + std::vector trackId; bool init_preprocessing(); @@ -168,7 +166,7 @@ public: CenternetDetection3DTrack() {}; ~CenternetDetection3DTrack() {}; bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); - void preprocess(cv::Mat &frame, const int bi=0, const std::vector& stream_size=std::vector()); + void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); }; diff --git a/include/tkDNN/DetectionNN3D.h b/include/tkDNN/DetectionNN3D.h index 7320bef..af6eaf9 100644 --- a/include/tkDNN/DetectionNN3D.h +++ b/include/tkDNN/DetectionNN3D.h @@ -54,7 +54,7 @@ class DetectionNN3D { * @param frame original frame to adapt for inference. * @param bi batch index */ - virtual void preprocess(cv::Mat &frame, const int bi=0 , const std::vector& stream_size=std::vector()) = 0; + virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; /** * This method postprocess the output of the NN to obtain the correct @@ -102,7 +102,7 @@ class DetectionNN3D { * box are needed, as in some cases for the mAP calculation. */ void update(std::vector& frames, const int cur_batches=1, bool save_times=false, - std::ofstream *times=nullptr, const bool mAP=false, const std::vector& stream_size=std::vector()){ + std::ofstream *times=nullptr, const bool mAP=false){ if(save_times && times==nullptr) FatalError("save_times set to true, but no valid ofstream given"); if(cur_batches > nBatches) @@ -116,7 +116,7 @@ class DetectionNN3D { if(!frames[bi].data) FatalError("No image data feed to detection"); originalSize.push_back(frames[bi].size()); - preprocess(frames[bi], bi, stream_size); + preprocess(frames[bi], bi); } TKDNN_TSTOP pre_stats.push_back(t_ns); diff --git a/src/CenternetDetection3D.cpp b/src/CenternetDetection3D.cpp index 73e4215..8f7d7c3 100644 --- a/src/CenternetDetection3D.cpp +++ b/src/CenternetDetection3D.cpp @@ -10,7 +10,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas classes = n_classes; nBatches = n_batches; confThreshold = conf_thresh; - + inputCalibs = k_calibs; dim = netRT->input_dim; const char *kitti_class_name[] = { @@ -99,19 +99,26 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas stddev << 0.229, 0.224, 0.225; #endif - calibs = cv::Mat(cv::Size(4,3), CV_32F); - calibs.at(0,0) = 707.0493; - calibs.at(0,1) = 0.0; - calibs.at(0,2) = 604.0814; - calibs.at(0,3) = 45.75831; - calibs.at(1,0) = 0.0; - calibs.at(1,1) = 707.0493; - calibs.at(1,2) = 180.5066; - calibs.at(1,3) = -0.3454157; - calibs.at(2,0) = 0.0; - calibs.at(2,1) = 0.0; - calibs.at(2,2) = 1.0; - calibs.at(2,3) = 0.004981016; + for(int bi=0; bi(0,0) = 707.0493; + calibs_.at(0,2) = 604.0814; + calibs_.at(1,1) = 707.0493; + calibs_.at(1,2) = 180.5066; + } + else { + calibs_.at(0,0) = inputCalibs[bi].at(0,0) * dim.w / 1440; + calibs_.at(0,2) = inputCalibs[bi].at(0,2) * dim.w / 1440; + calibs_.at(1,1) = inputCalibs[bi].at(1,1) * dim.h / 1080; + calibs_.at(1,2) = inputCalibs[bi].at(1,2) * dim.h / 1080; + } + calibs_.at(0,3) = 45.75831; + calibs_.at(1,3) = -0.3454157; + calibs_.at(2,2) = 1.0; + calibs_.at(2,3) = 0.004981016; + calibs.push_back(calibs_); + } r = cv::Mat(cv::Size(3,3), CV_32F); r.at(0,1) = 0.0; @@ -139,8 +146,8 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) ); // Alloc array used in the kernel - checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); - checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); + checkCuda( cudaMalloc(&srcOut, K *sizeof(float)) ); + checkCuda( cudaMalloc(&idsOut, K *sizeof(int)) ); dst2.at(0,0)=width * 0.5; dst2.at(0,1)=width * 0.5; @@ -150,16 +157,14 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); - face_id.push_back({0,1,5,4}); - face_id.push_back({1,2,6, 5}); - face_id.push_back({2,3,7,6}); - face_id.push_back({3,0,4,7}); + faceId.push_back({0,1,5,4}); + faceId.push_back({1,2,6, 5}); + faceId.push_back({2,3,7,6}); + faceId.push_back({3,0,4,7}); // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } -void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi, const std::vector& stream_size){ - // -----------------------------------pre-process ------------------------------------------ - +void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){ // auto start_t = std::chrono::steady_clock::now(); // auto step_t = std::chrono::steady_clock::now(); // auto end_t = std::chrono::steady_clock::now(); @@ -338,20 +343,20 @@ void CenternetDetection3D::postprocess(const int bi, const bool mAP) { // ----------- topk end - topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out); + topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], srcOut, idsOut); // checkCuda( cudaDeviceSynchronize() ); - getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[4], dep_d, ids_out); + getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[4], dep_d, idsOut); checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) ); - getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[5], rot_d, ids_out); + getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[5], rot_d, idsOut); checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) ); - getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[6], dim_d, ids_out); + getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[6], dim_d, idsOut); checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) ); - getRecordsFromTopKId(topk_inds_d, K, dim_wh.c, dim_wh.h * dim_wh.w, rt_out[2], wh_d, ids_out); + getRecordsFromTopKId(topk_inds_d, K, dim_wh.c, dim_wh.h * dim_wh.w, rt_out[2], wh_d, idsOut); checkCuda( cudaMemcpy(wh, wh_d, K * dim_wh.c * sizeof(float), cudaMemcpyDeviceToHost) ); checkCuda( cudaMemcpy(xs, topk_xs_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); @@ -397,11 +402,11 @@ void CenternetDetection3D::postprocess(const int bi, const bool mAP) { alpha = std::atan2(rot[6*K + j], rot[7*K + j]) +0.5 * M_PI; // unproject_2d_to_3d - z = dep[j] - calibs.at(2,3);// z = depth - P[2, 3] - x = (target_coords[j*4] * dep[j] - calibs.at(0,3) - calibs.at(0,2) * z) / calibs.at(0,0); - y = (target_coords[j*4+1] * dep[j] - calibs.at(1,3) - calibs.at(1,2) * z) / calibs.at(1,1) + (dim_[j] / 2); + z = dep[j] - calibs[bi].at(2,3);// z = depth - P[2, 3] + x = (target_coords[j*4] * dep[j] - calibs[bi].at(0,3) - calibs[bi].at(0,2) * z) / calibs[bi].at(0,0); + y = (target_coords[j*4+1] * dep[j] - calibs[bi].at(1,3) - calibs[bi].at(1,2) * z) / calibs[bi].at(1,1) + (dim_[j] / 2); // alpha2rot_y - rot_y = (alpha + std::atan2(target_coords[j*4] - calibs.at(0,2), calibs.at(0,0))); + rot_y = (alpha + std::atan2(target_coords[j*4] - calibs[bi].at(0,2), calibs[bi].at(0,0))); if(rot_y>M_PI) rot_y -= 2*M_PI; if(rot_y(k1,k2) = aus.at(k1,k2); } aus.release(); - aus = calibs * pts3DHomo; + aus = calibs[bi] * pts3DHomo; tk::dnn::box3D res; for(int k=0; k<8; k++) { @@ -486,31 +491,31 @@ void CenternetDetection3D::draw(std::vector& frames) { for(int ind_f = 3; ind_f>=0; ind_f--) { for(int j=0; j<4; j++) { - cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), - b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(j) * 2), + b.corners.at(faceId.at(ind_f).at(j) * 2 + 1)), + cv::Point(b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2), + b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1)), colors[b.cl], 2); if(ind_f == 0) { - cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), - b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), - b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); - cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), - b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), - b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(0) * 2), + b.corners.at(faceId.at(ind_f).at(0) * 2 + 1)), + cv::Point(b.corners.at(faceId.at(ind_f).at(2) * 2), + b.corners.at(faceId.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); + cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(1) * 2), + b.corners.at(faceId.at(ind_f).at(1) * 2 + 1)), + cv::Point(b.corners.at(faceId.at(ind_f).at(3) * 2), + b.corners.at(faceId.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); } } } // draw label cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); - cv::rectangle(frames[bi], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1)), - cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), - (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); - cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), + cv::rectangle(frames[bi], cv::Point(b.corners.at(faceId.at(0).at(0) * 2), + b.corners.at(faceId.at(0).at(0) * 2 + 1)), + cv::Point((b.corners.at(faceId.at(0).at(0) * 2) + text_size.width - 2), + (b.corners.at(faceId.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); + cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(faceId.at(0).at(0) * 2), + b.corners.at(faceId.at(0).at(0) * 2 + 1) - (baseline / 2)), cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); } } diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index 3653896..d119c1e 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -6,83 +6,77 @@ namespace tk { namespace dnn { bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh, const std::vector& k_calibs) { - netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); - - dim = netRT->input_dim; - dim.c = 3; - nBatches = n_batches; + netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); + dim = netRT->input_dim; + dim.c = 3; + nBatches = n_batches; confThreshold = conf_thresh; - inputCalibs = k_calibs; - tr_res.resize(nBatches); - count_tr.resize(nBatches, 0); + inputCalibs = k_calibs; init_preprocessing(); init_pre_inf(); init_postprocessing(); init_visualization(n_classes); - } bool CenternetDetection3DTrack::init_preprocessing(){ //image transformation - src = cv::Mat(cv::Size(2,3), CV_32F); - dst = cv::Mat(cv::Size(2,3), CV_32F); - dst2 = cv::Mat(cv::Size(2,3), CV_32F); - trans = cv::Mat(cv::Size(3,2), CV_32F); - trans2 = cv::Mat(cv::Size(3,2), CV_32F); - trans_out = cv::Mat(cv::Size(3,2), CV_32F); + src = cv::Mat(cv::Size(2,3), CV_32F); + dst = cv::Mat(cv::Size(2,3), CV_32F); + dst2 = cv::Mat(cv::Size(2,3), CV_32F); + trans = cv::Mat(cv::Size(3,2), CV_32F); + trans2 = cv::Mat(cv::Size(3,2), CV_32F); + transOut = cv::Mat(cv::Size(3,2), CV_32F); - - dst2.at(0,0)=width * 0.5; - dst2.at(0,1)=width * 0.5; - dst2.at(1,0)=width * 0.5; - dst2.at(1,1)=width * 0.5 + width * -0.5; - - dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); - dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); + dst2.at(0,0) = width * 0.5; + dst2.at(0,1) = width * 0.5; + dst2.at(1,0) = width * 0.5; + dst2.at(1,1) = width * 0.5 + width * -0.5; + dst2.at(2,0) = dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); + dst2.at(2,1) = dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); for(int bi=0; biinput_dim.tot() * nBatches)); - checkCuda(cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); + checkCuda( cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); + checkCuda( cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); } bool CenternetDetection3DTrack::init_pre_inf(){ // initial steps: the first part of the network const char *pre_img_conv1_bin = "dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; - const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; - const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; - const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; + const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; + const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; + const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; dim_in0 = tk::dnn::dataDim_t(1, 3, 512, 512, 1); dim_in1 = tk::dnn::dataDim_t(1, 1, 512, 512, 1); - checkCuda( cudaMalloc(&out_d, netRT->input_dim.tot()*sizeof(dnnType)) ); checkCuda( cudaMalloc(&img_d, dim_in0.tot()*sizeof(dnnType)) ); checkCuda( cudaMalloc(&hm_d, dim_in1.tot()*sizeof(dnnType)) ); // init to zeros hm - dnnType *hm_h; + dnnType *hm_h; checkCuda( cudaMallocHost(&hm_h, 1 * dim.h * dim.w*sizeof(dnnType)) ); for(int i=0; i<1 * dim.h * dim.w; i++) - hm_h[i]=0.0f; + hm_h[i] = 0.0f; checkCuda( cudaMemcpy(hm_d, hm_h, 1 * dim.h * dim.w * sizeof(dnnType), cudaMemcpyHostToDevice) ); checkCuda( cudaFreeHost(hm_h) ); dnnType *i0_h, *i1_h, *i2_h; @@ -97,20 +91,20 @@ bool CenternetDetection3DTrack::init_pre_inf(){ pre_phase_net = new tk::dnn::Network(dim_in0); //pre-img - tk::dnn::Input *in_pre_img = new tk::dnn::Input(pre_phase_net, dim_in0, img_d); - tk::dnn::Conv2d *pre_img_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true); - tk::dnn::Activation *pre_img_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + tk::dnn::Input *in_pre_img = new tk::dnn::Input(pre_phase_net, dim_in0, img_d); + tk::dnn::Conv2d *pre_img_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true); + tk::dnn::Activation *pre_img_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); //pre-hm - tk::dnn::Input *in_pre_hm = new tk::dnn::Input(pre_phase_net, dim_in1, hm_d); - tk::dnn::Conv2d *pre_hm_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true); - tk::dnn::Activation *pre_hm_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + tk::dnn::Input *in_pre_hm = new tk::dnn::Input(pre_phase_net, dim_in1, hm_d); + tk::dnn::Conv2d *pre_hm_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true); + tk::dnn::Activation *pre_hm_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); // image input - tk::dnn::Input *input_image = new tk::dnn::Input(pre_phase_net, dim_in0, input_pre_inf_d); - tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); - tk::dnn::Activation *relu1 = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); + tk::dnn::Input *input_image = new tk::dnn::Input(pre_phase_net, dim_in0, input_pre_inf_d); + tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation *relu1 = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU); - tk::dnn::Shortcut *s0_input = new tk::dnn::Shortcut(pre_phase_net, pre_img_relu); - tk::dnn::Shortcut *s1_input = new tk::dnn::Shortcut(pre_phase_net, pre_hm_relu); + tk::dnn::Shortcut *s0_input = new tk::dnn::Shortcut(pre_phase_net, pre_img_relu); + tk::dnn::Shortcut *s1_input = new tk::dnn::Shortcut(pre_phase_net, pre_hm_relu); // output data out_d = s1_input->dstData; //print network model @@ -123,14 +117,14 @@ bool CenternetDetection3DTrack::init_pre_inf(){ bool CenternetDetection3DTrack::init_postprocessing(){ srand(0); //seed = 0 for random colors - dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1); - dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); - dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); - dim_track = tk::dnn::dataDim_t(1, 2, 128, 128, 1); - dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1); - dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1); - dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1); - dim_amodel_offset = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1); + dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_track = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1); + dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1); + dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1); + dim_amodel_offset = tk::dnn::dataDim_t(1, 2, 128, 128, 1); checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) ); checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) ); @@ -138,7 +132,7 @@ bool CenternetDetection3DTrack::init_postprocessing(){ checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) ); checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) ); - for(int i =0; i(coco_class_name, std::end( coco_class_name)); for(int c=0; c(3,6) = 1.0; pts3DHomo.at(3,7) = 1.0; - face_id.push_back({0,1,5,4}); - face_id.push_back({1,2,6, 5}); - face_id.push_back({3,0,4,7}); - face_id.push_back({2,3,7,6}); + faceId.push_back({0,1,5,4}); + faceId.push_back({1,2,6, 5}); + faceId.push_back({3,0,4,7}); + faceId.push_back({2,3,7,6}); // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } @@ -296,22 +289,21 @@ void CenternetDetection3DTrack::pre_inf(const int bi){ checkCuda( cudaDeviceSynchronize() ); } -void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const std::vector& stream_size){ - // -----------------------------------pre-process ------------------------------------------ +void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ cv::Size sz = originalSize[bi]; - float scale = 1.0; - float new_height = sz.height * scale; - float new_width = sz.width * scale; - if(sz.height != sz_old[bi].height && sz.width != sz_old[bi].width){ + // float scale = 1.0; + float new_height = dim.h;//sz.height * scale; + float new_width = dim.w;//sz.width * scale; + if(sz.height != szOld[bi].height && sz.width != szOld[bi].width){ if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) { calibs[bi].at(0,2) = new_width / 2.0f; calibs[bi].at(1,2) = new_height /2.0f; } else { - calibs[bi].at(0,0) = inputCalibs[bi].at(0,0) * dim.w / stream_size[bi].width; - calibs[bi].at(0,2) = inputCalibs[bi].at(0,2) * dim.w / stream_size[bi].width; - calibs[bi].at(1,1) = inputCalibs[bi].at(1,1) * dim.h / stream_size[bi].height; - calibs[bi].at(1,2) = inputCalibs[bi].at(1,2) * dim.h / stream_size[bi].height; + calibs[bi].at(0,0) = inputCalibs[bi].at(0,0) * dim.w / sz.width; + calibs[bi].at(0,2) = inputCalibs[bi].at(0,2) * dim.w / sz.width; + calibs[bi].at(1,1) = inputCalibs[bi].at(1,1) * dim.h / sz.height; + calibs[bi].at(1,2) = inputCalibs[bi].at(1,2) * dim.h / sz.height; } float c[] = {new_width / 2.0f, new_height /2.0f}; @@ -320,34 +312,33 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const s // ----------- get_affine_transform // rot_rad = pi * 0 / 100 --> 0 //dim.print(); - src.at(0,0)=c[0]; - src.at(0,1)=c[1]; - src.at(1,0)=c[0]; - src.at(1,1)=c[1] + s[0] * -0.5; - dst.at(0,0)=dim.w * 0.5; - dst.at(0,1)=dim.h * 0.5; - dst.at(1,0)=dim.w * 0.5; - dst.at(1,1)=dim.h * 0.5 + dim.w * -0.5; + src.at(0,0) = c[0]; + src.at(0,1) = c[1]; + src.at(1,0) = c[0]; + src.at(1,1) = c[1] + s[0] * -0.5; + dst.at(0,0) = dim.w * 0.5; + dst.at(0,1) = dim.h * 0.5; + dst.at(1,0) = dim.w * 0.5; + dst.at(1,1) = dim.h * 0.5 + dim.w * -0.5; - src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); - src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); - dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); - dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); + src.at(2,0) = src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); + src.at(2,1) = src.at(1,1) + (src.at(0,0)-src.at(1,0) ); + dst.at(2,0) = dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); + dst.at(2,1) = dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); trans = cv::getAffineTransform( src, dst ); trans2 = cv::getAffineTransform( dst2, src ); - trans2.convertTo(trans_out, CV_32F); + trans2.convertTo(transOut, CV_32F); } - sz_old[bi] = sz; + szOld[bi] = sz; #ifdef OPENCV_CUDACONTRIB - std::cout<<"OPENCV CPMTROB\n"; cv::cuda::GpuMat im_Orig; cv::cuda::GpuMat imageF1_d, imageF2_d; im_Orig = cv::cuda::GpuMat(frame); - // cv::cuda::resize (im_Orig, imageF1_d, cv::Size(new_width, new_height)); - imageF1_d = im_Orig; + cv::cuda::resize (im_Orig, imageF1_d, cv::Size(dim.w, dim.h)); + // imageF1_d = im_Orig; checkCuda( cudaDeviceSynchronize() ); sz = imageF1_d.size(); @@ -367,20 +358,18 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const s normalize(d_ptrs, dim.c, dim.h, dim.w, mean_d, stddev_d); - checkCuda(cudaMemcpy(input_pre_inf_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + checkCuda( cudaMemcpy(input_pre_inf_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); checkCuda( cudaDeviceSynchronize() ); #else - std::cout<<"NO OPENCV CPMTROB\n"; cv::Mat imageF; - //resize(frame, imageF, cv::Size(512, 512)); - imageF = frame; + resize(frame, imageF, cv::Size(dim.w, dim.h)); + // imageF = frame; sz = imageF.size(); - cv::warpAffine(imageF, imageF, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR ); - //cv::imshow("warp", imageF); - + // cv::imshow("warp", imageF); + sz = imageF.size(); imageF.convertTo(imageF, CV_32FC3, 1/255.0); @@ -394,11 +383,11 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi, const s bgr[i] = bgr[i] / stddev[i]; } for(int i=0; i(0,0) = x1; target_coords.at(0,1) = x2; target_coords.at(0,2) = 1.0; - return trans_out * target_coords; + return transOut * target_coords; } void CenternetDetection3DTrack::tracking(const int bi) { - float item_size[count_det]; - int item_cl[count_det]; - float dets[2*count_det]; - for(int i=0; i(0,0) - det_res[i].bb0.at(0,0)) * - (det_res[i].bb1.at(0,1) - det_res[i].bb0.at(0,1)); - item_cl[i] = det_res[i].cl; - dets[i*2] = det_res[i].ct.at(0,0); - dets[i*2+1] = det_res[i].ct.at(0,1); + float item_size[countDet]; + int item_cl[countDet]; + float dets[2*countDet]; + for(int i=0; i(0,0) - detRes[i].bb0.at(0,0)) * + (detRes[i].bb1.at(0,1) - detRes[i].bb0.at(0,1)); + item_cl[i] = detRes[i].cl; + dets[i*2] = detRes[i].ct.at(0,0); + dets[i*2+1] = detRes[i].ct.at(0,1); } - float track_size[count_tr[bi]]; - int track_cl[count_tr[bi]]; - float tracks[2*count_tr[bi]]; - for(int i=0; i(0,0) - tr_res[bi][i].det_res.bb0.at(0,0)) * - (tr_res[bi][i].det_res.bb1.at(0,1) - tr_res[bi][i].det_res.bb0.at(0,1)); - track_cl[i] = tr_res[bi][i].det_res.cl; - tracks[i*2] = tr_res[bi][i].det_res.ct.at(0,0); - tracks[i*2+1] = tr_res[bi][i].det_res.ct.at(0,1); + float track_size[countTr[bi]]; + int track_cl[countTr[bi]]; + float tracks[2*countTr[bi]]; + for(int i=0; i(0,0) - trRes[bi][i].det_res.bb0.at(0,0)) * + (trRes[bi][i].det_res.bb1.at(0,1) - trRes[bi][i].det_res.bb0.at(0,1)); + track_cl[i] = trRes[bi][i].det_res.cl; + tracks[i*2] = trRes[bi][i].det_res.ct.at(0,0); + tracks[i*2+1] = trRes[bi][i].det_res.ct.at(0,1); } - float dist[count_tr[bi]*count_det]; + float dist[countTr[bi]*countDet]; bool invalid; - for(int i=0; i track_size[i] || dist[j*count_tr[bi]+i] > item_size[j] || item_cl[j] != track_cl[i]; - dist[j*count_tr[bi]+i] = dist[j*count_tr[bi]+i] + invalid * (1 << 18); + for(int i=0; i track_size[i] || + dist[j*countTr[bi]+i] > item_size[j] || + item_cl[j] != track_cl[i]; + dist[j*countTr[bi]+i] = dist[j*countTr[bi]+i] + invalid * (1 << 18); } } - int matched_indices[2*count_tr[bi]]; + int matched_indices[2*countTr[bi]]; float min_tr; - int min_idtr=-1; - for(int i=0; i new_tr_res; int id_new_tr=0; - for(int i=0; i new_thresh) { + int count_tr_ = countTr[bi]; + for(int i=0; i newThresh) { count_tr_ ++; struct trackingRes new_tr_res_; - new_tr_res_.det_res.score = det_res[i].score; - new_tr_res_.det_res.cl = det_res[i].cl; - new_tr_res_.det_res.ct = det_res[i].ct; - new_tr_res_.det_res.tr = det_res[i].tr; - new_tr_res_.det_res.bb0 = det_res[i].bb0; - new_tr_res_.det_res.bb1 = det_res[i].bb1; - new_tr_res_.det_res.dep = det_res[i].dep; - new_tr_res_.det_res.dim[0] = det_res[i].dim[0]; - new_tr_res_.det_res.dim[1] = det_res[i].dim[1]; - new_tr_res_.det_res.dim[2] = det_res[i].dim[2]; - new_tr_res_.det_res.alpha = det_res[i].alpha; - new_tr_res_.det_res.x = det_res[i].x; - new_tr_res_.det_res.y = det_res[i].y; - new_tr_res_.det_res.z = det_res[i].z; - new_tr_res_.det_res.rot_y = det_res[i].rot_y; - new_tr_res_.tracking_id = track_id[bi]++; - new_tr_res_.age = 1; - new_tr_res_.active = 1; - new_tr_res_.color = rand() % 256; - if(tr_res.size() <= bi) { + new_tr_res_.det_res.score = detRes[i].score; + new_tr_res_.det_res.cl = detRes[i].cl; + new_tr_res_.det_res.ct = detRes[i].ct; + new_tr_res_.det_res.tr = detRes[i].tr; + new_tr_res_.det_res.bb0 = detRes[i].bb0; + new_tr_res_.det_res.bb1 = detRes[i].bb1; + new_tr_res_.det_res.dep = detRes[i].dep; + new_tr_res_.det_res.dim[0] = detRes[i].dim[0]; + new_tr_res_.det_res.dim[1] = detRes[i].dim[1]; + new_tr_res_.det_res.dim[2] = detRes[i].dim[2]; + new_tr_res_.det_res.alpha = detRes[i].alpha; + new_tr_res_.det_res.x = detRes[i].x; + new_tr_res_.det_res.y = detRes[i].y; + new_tr_res_.det_res.z = detRes[i].z; + new_tr_res_.det_res.rot_y = detRes[i].rot_y; + new_tr_res_.tracking_id = trackId[bi]++; + new_tr_res_.age = 1; + new_tr_res_.active = 1; + new_tr_res_.color = rand() % 256; + if(trRes.size() <= bi) { std::vector v_new_tr_res_; v_new_tr_res_.push_back(new_tr_res_); - tr_res.push_back(v_new_tr_res_); + trRes.push_back(v_new_tr_res_); } else - tr_res[bi].push_back(new_tr_res_); + trRes[bi].push_back(new_tr_res_); } } - count_tr[bi] = count_tr_; - - if(track_id[bi]==1000) - track_id[bi]=0; - det_res.clear(); + countTr[bi] = count_tr_; + //reset the tracker id + if(trackId[bi] == 1000) + trackId[bi] = 0; + detRes.clear(); } @@ -697,35 +686,36 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { // ---------------------------------- post-process ----------------------------------------- - count_det = 0; - det_res.clear(); - for(int i = 0; i(2,3); - new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs[bi].at(0,3) - calibs[bi].at(0,2) * new_det_res.z) / calibs[bi].at(0,0); - new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs[bi].at(1,3) - calibs[bi].at(1,2) * new_det_res.z) / calibs[bi].at(1,1) + (dim_[i] / 2); + new_det_res.x = ((float)new_det_res.ct.at(0,0) * dep[i] - calibs[bi].at(0,3) - + calibs[bi].at(0,2) * new_det_res.z) / calibs[bi].at(0,0); + new_det_res.y = ((float)new_det_res.ct.at(0,1) * dep[i] - calibs[bi].at(1,3) - + calibs[bi].at(1,2) * new_det_res.z) / calibs[bi].at(1,1) + (dim_[i] / 2); // alpha2rot_y // idx = rot[:, 1] > rot[:, 5] @@ -737,13 +727,11 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { else new_det_res.alpha = std::atan2(rot[6*K + i], rot[7*K + i]) +0.5 * M_PI; new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at(0,0) - calibs[bi].at(0,2), calibs[bi].at(0,0))); - new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest - det_res.push_back(new_det_res); - + new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest + detRes.push_back(new_det_res); } // track step tracking(bi); - batchTracked.push_back(tr_res[bi]); } void CenternetDetection3DTrack::draw(std::vector& frames) { @@ -754,32 +742,38 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { int baseline = 0; float font_scale = 0.8; int thickness = 2; + for(int bi=0; bi vis_thresh){// && t.active!=0) { + if(t.det_res.score > confThreshold){// && t.active!=0) { if(view2d) { - cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), t.det_res.bb0.at(0,1)), - cv::Point(t.det_res.bb1.at(0,0), t.det_res.bb1.at(0,1)), tr_colors[t.color], thickness); - cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), - t.det_res.bb0.at(0,1) - text_size.height - thickness), - cv::Point(t.det_res.bb0.at(0,0) + text_size.width, - t.det_res.bb0.at(0,1)), tr_colors[t.color], -1); + cv::rectangle(frames[bi], + cv::Point(t.det_res.bb0.at(0,0) * scale_x, t.det_res.bb0.at(0,1) * scale_y), + cv::Point(t.det_res.bb1.at(0,0) * scale_x, t.det_res.bb1.at(0,1) * scale_y), + trColors[t.color], thickness); + cv::rectangle(frames[bi], + cv::Point(t.det_res.bb0.at(0,0) * scale_x, t.det_res.bb0.at(0,1) * scale_y - text_size.height - thickness), + cv::Point(t.det_res.bb0.at(0,0) * scale_x + text_size.width, t.det_res.bb0.at(0,1) * scale_y), + trColors[t.color], -1); - cv::putText(frames[bi], txt, cv::Point(t.det_res.bb0.at(0,0), - t.det_res.bb0.at(0,1) - thickness -1), - cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); + cv::putText(frames[bi], txt, + cv::Point(t.det_res.bb0.at(0,0) * scale_x, t.det_res.bb0.at(0,1) * scale_y - thickness -1), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frames[bi], cv::Point((int)t.det_res.ct.at(0,0), - (int)t.det_res.ct.at(0,1)), - cv::Point((int)(t.det_res.ct.at(0,0) + t.det_res.tr.at(0,0)), - (int)(t.det_res.ct.at(0,1) + t.det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); + cv::arrowedLine(frames[bi], + cv::Point((int)t.det_res.ct.at(0,0) * scale_x, (int)t.det_res.ct.at(0,1) * scale_y), + cv::Point((int)(t.det_res.ct.at(0,0) * scale_x + t.det_res.tr.at(0,0) * scale_x), + (int)(t.det_res.ct.at(0,1) * scale_y + t.det_res.tr.at(0,1) * scale_y)), + cv::Scalar(255, 0, 255), 2); } //3d if(!view2d && t.det_res.z > 1){ @@ -833,50 +827,59 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { res_corners.push_back(aus.at(1,k) / aus.at(2,k)); } aus.release(); - for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int ind_f=3; ind_f>=0; ind_f--) { for(int j=0; j<4; j++) { - cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), - (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), - tr_colors[t.color], 2); + cv::line(frames[bi], + cv::Point((int)res_corners.at(faceId.at(ind_f).at(j) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(j) * 2 + 1) * scale_y), + cv::Point((int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1) * scale_y), + trColors[t.color], 2); if(ind_f == 0 && j==3) { - cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), - (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), - (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[t.color], 2); - cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), - (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), - (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[t.color], 2); + cv::line(frames[bi], + cv::Point((int)res_corners.at(faceId.at(ind_f).at(0) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(0) * 2 + 1) * scale_y), + cv::Point((int)res_corners.at(faceId.at(ind_f).at(2) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(2) * 2 + 1) * scale_y), trColors[t.color], 2); + cv::line(frames[bi], + cv::Point((int)res_corners.at(faceId.at(ind_f).at(1) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(1) * 2 + 1) * scale_y), + cv::Point((int)res_corners.at(faceId.at(ind_f).at(3) * 2) * scale_x, + (int)res_corners.at(faceId.at(ind_f).at(3) * 2 + 1) * scale_y), trColors[t.color], 2); } } } float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; for(int k=0; k<8; k++) { - if(res_corners[2*k]bb1) - bb1=res_corners[2*k]; - if(res_corners[2*k+1]bb3) - bb3=res_corners[2*k+1]; + if(res_corners[2*k] < bb0) + bb0 = res_corners[2*k]; + if(res_corners[2*k] > bb1) + bb1 = res_corners[2*k]; + if(res_corners[2*k+1] < bb2) + bb2 = res_corners[2*k+1]; + if(res_corners[2*k+1] > bb3) + bb3 = res_corners[2*k+1]; } // if(not no_bbox): - // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), - // tr_colors[t.color], thickness); - cv::rectangle(frames[bi], cv::Point(bb0, bb2 - text_size.height - thickness), - cv::Point(bb0 + text_size.width, bb2), tr_colors[t.color], -1); + // cv::rectangle(frame, + // cv::Point(bb0, bb2), + // cv::Point(bb1, bb3), + // trColors[t.color], thickness); + cv::rectangle(frames[bi], + cv::Point(bb0 * scale_x, bb2 * scale_y - text_size.height - thickness), + cv::Point(bb0 * scale_x + text_size.width, bb2 * scale_y), + trColors[t.color], -1); - cv::putText(frames[bi], txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, - font_scale, cv::Scalar(255, 255, 255), 1); + cv::putText(frames[bi], txt, + cv::Point(bb0 * scale_x, bb2 * scale_y - thickness -1), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frames[bi], cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), - cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at(0,0)), - (int)((bb2 + bb3)/2 + t.det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); + cv::arrowedLine(frames[bi], + cv::Point((int)((bb0 + bb1)/2) * scale_x, (int)((bb2 + bb3)/2) * scale_y), + cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at(0,0)) * scale_x, + (int)((bb2 + bb3)/2 + t.det_res.tr.at(0,1)) * scale_y), + cv::Scalar(255, 0, 255), 2); } } } -- 2.52.0 From 34c1c3d577cb55235f0c73a5eee201023fc0530b Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 11 May 2021 16:17:23 +0200 Subject: [PATCH 131/228] Update cnet branch. This commit splits the demo3D in two demo: one for the 3D object detection and one for the tracking. It renames the files related to CenterTrack. It adds a new parameter to select the tracker mode (2D or 3D). Signed-off-by: Davide Sapienza --- CMakeLists.txt | 13 +- demo/demo/demo3D.cpp | 5 - demo/demo/demoTracker.cpp | 157 +++++++++++++ ...ternetDetection3DTrack.h => CenterTrack.h} | 20 +- include/tkDNN/TrackingNN.h | 158 +++++++++++++ ...etDetection3DTrack.cpp => CenterTrack.cpp} | 47 ++-- .../dla34_ctrack/dla34_ctrack.cpp} | 222 +++++++++--------- 7 files changed, 469 insertions(+), 153 deletions(-) create mode 100644 demo/demo/demoTracker.cpp rename include/tkDNN/{CenternetDetection3DTrack.h => CenterTrack.h} (90%) create mode 100644 include/tkDNN/TrackingNN.h rename src/{CenternetDetection3DTrack.cpp => CenterTrack.cpp} (96%) rename tests/{centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp => centertrack/dla34_ctrack/dla34_ctrack.cpp} (73%) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb2b1c6..197dced 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,9 +46,9 @@ include_directories(${EIGEN3_INCLUDE_DIR}) find_package(OpenCV REQUIRED) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV") -if(OpenCV_CUDA_VERSION) - add_compile_definitions(OPENCV_CUDACONTRIB) -endif() +# if(OpenCV_CUDA_VERSION) +# add_compile_definitions(OPENCV_CUDACONTRIB) +# endif() # gives problems in cross-compiling, probably malformed cmake config find_package(yaml-cpp REQUIRED) @@ -120,8 +120,8 @@ target_link_libraries(test_resnet101_cnet3d tkDNN) add_executable(test_dla34_cnet3d tests/centernet/dla34_cnet3d/dla34_cnet3d.cpp) target_link_libraries(test_dla34_cnet3d tkDNN) -add_executable(test_dla34_cnet3d_track tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp) -target_link_libraries(test_dla34_cnet3d_track tkDNN) +add_executable(test_dla34_ctrack tests/centertrack/dla34_ctrack/dla34_ctrack.cpp) +target_link_libraries(test_dla34_ctrack tkDNN) # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) @@ -136,6 +136,9 @@ target_link_libraries(demo tkDNN) add_executable(demo3D demo/demo/demo3D.cpp) target_link_libraries(demo3D tkDNN) +add_executable(demoTracker demo/demo/demoTracker.cpp) +target_link_libraries(demoTracker tkDNN) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 620b0d4..90d4bfb 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -5,7 +5,6 @@ #include #include "CenternetDetection3D.h" -#include "CenternetDetection3DTrack.h" bool gRun; bool SAVE_RESULT = false; @@ -55,7 +54,6 @@ int main(int argc, char *argv[]) { SAVE_RESULT = true; tk::dnn::CenternetDetection3D cnet; - tk::dnn::CenternetDetection3DTrack ctrack; tk::dnn::DetectionNN3D *detNN; @@ -64,9 +62,6 @@ int main(int argc, char *argv[]) { case 'c': detNN = &cnet; break; - case 't': - detNN = &ctrack; - break; default: FatalError("Network type not allowed (3rd parameter)\n"); } diff --git a/demo/demo/demoTracker.cpp b/demo/demo/demoTracker.cpp new file mode 100644 index 0000000..ad6e204 --- /dev/null +++ b/demo/demo/demoTracker.cpp @@ -0,0 +1,157 @@ +#include +#include +#include /* srand, rand */ +//#include +#include + +#include "CenterTrack.h" + +bool gRun; +bool SAVE_RESULT = false; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + std::cout<<"detection\n"; + signal(SIGINT, sig_handler); + + + std::string net = "dla34_cnet3d_track_fp32.rt"; + if(argc > 1) + net = argv[1]; + #ifdef __linux__ + std::string input = "../demo/yolo_test.mp4"; + #elif _WIN32 + std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + #endif + + if(argc > 2) + input = argv[2]; + char ntype = 'c'; + if(argc > 3) + ntype = argv[3][0]; + int n_classes = 3; + if(argc > 4) + n_classes = atoi(argv[4]); + int n_batch = 1; + if(argc > 5) + n_batch = atoi(argv[5]); + bool show = true; + if(argc > 6) + show = atoi(argv[6]); + float conf_thresh=0.3; + if(argc > 7) + conf_thresh = atof(argv[7]); + bool t3d = true; + if(argc > 8) + t3d = atoi(argv[8]); + if(n_batch < 1 || n_batch > 64) + FatalError("Batch dim not supported"); + + if(!show) + SAVE_RESULT = true; + + tk::dnn::CenterTrack ctrack; + + tk::dnn::TrackingNN *trackNN; + + switch(ntype) + { + case 'c': + trackNN = &ctrack; + break; + default: + FatalError("Network type not allowed (3rd parameter)\n"); + } + std::vector calibs; + // cv::Mat calib = cv::Mat::zeros(cv::Size(3,3), CV_32F); + // calib.at(0,0) = 864.1243196486207;// * 512.0;//884.081444212;//864.1243196486207 * 512.0;// 633.0; + // calib.at(0,2) = 726.7271690557819;// * 512.0;//0.0;//726.7271690557819 * 512.0;// 0.0; //w/2 + // calib.at(1,1) = 883.6552349216504;// * 512.0;//884.081444212;//883.6552349216504 * 512.0;// 633.0; + // calib.at(1,2) = 506.8548506986564;// * 512.0;//0.0;//506.8548506986564 * 512.0;// 0.0; //h/2 + // calibs.push_back(calib); + // calibs.push_back(calib); + // calibs.push_back(calib); + // calibs.push_back(calib); + trackNN->init(net, n_classes, n_batch, conf_thresh, t3d, calibs); + + gRun = true; + + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + cv::Mat frame; + if(show) + cv::namedWindow("detection", cv::WINDOW_NORMAL); + + std::vector batch_frame; + std::vector batch_dnn_input; + + while(gRun) { + batch_dnn_input.clear(); + batch_frame.clear(); + + for(int bi=0; bi< n_batch; ++bi){ + cap >> frame; + if(!frame.data) + break; + batch_frame.push_back(frame); + + // this will be resized to the net format + batch_dnn_input.push_back(frame.clone()); + } + if(!frame.data) + break; + + //inference + trackNN->update(batch_dnn_input, n_batch, false, nullptr, false); + trackNN->draw(batch_frame); + + if(show){ + for(int bi=0; bi< n_batch; ++bi){ + cv::imshow("detection", batch_frame[bi]); + cv::waitKey(1); + } + } + if(n_batch == 1 && SAVE_RESULT) + resultVideo << frame; + } + + std::cout<<"detection end\n"; + double mean = 0; + + std::cout<pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(trackNN->pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n"; + for(int i=0; ipre_stats.size(); i++) mean += trackNN->pre_stats[i]; mean /= trackNN->pre_stats.size(); + std::cout<<"Avg: "<stats.begin(), trackNN->stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(trackNN->stats.begin(), trackNN->stats.end())<<" ms\n"; + for(int i=0; istats.size(); i++) mean += trackNN->stats[i]; mean /= trackNN->stats.size(); + std::cout<<"Avg: "<post_stats.begin(), trackNN->post_stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(trackNN->post_stats.begin(), trackNN->post_stats.end())<<" ms\n"; + for(int i=0; ipost_stats.size(); i++) mean += trackNN->post_stats[i]; mean /= trackNN->post_stats.size(); + std::cout<<"Avg: "< #include "opencv2/opencv.hpp" @@ -11,7 +11,7 @@ #include // std::iota #include // std::sort -#include "DetectionNN3D.h" +#include "TrackingNN.h" #include "kernelsThrust.h" @@ -49,7 +49,7 @@ struct trackingRes int color; }; -class CenternetDetection3DTrack : public DetectionNN3D +class CenterTrack : public TrackingNN { public: tk::dnn::dataDim_t dim; @@ -133,7 +133,7 @@ public: std::vector> faceId; cv::Scalar trColors[256]; - bool view2d = false; + bool mode3D; //processing struct threshold op; @@ -163,9 +163,11 @@ public: public: tk::dnn::Network *pre_phase_net = nullptr; - CenternetDetection3DTrack() {}; - ~CenternetDetection3DTrack() {}; - bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector& k_calibs=std::vector()); + CenterTrack() {}; + ~CenterTrack() {}; + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, + const float conf_thresh=0.3, const bool mode_3d=true, + const std::vector& k_calibs=std::vector()); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); void draw(std::vector& frames); @@ -176,4 +178,4 @@ public: } // namespace tk -#endif /*CENTERNETDETECTION3DTRACK_H*/ \ No newline at end of file +#endif /*CENTERTRACK_H*/ \ No newline at end of file diff --git a/include/tkDNN/TrackingNN.h b/include/tkDNN/TrackingNN.h new file mode 100644 index 0000000..476db53 --- /dev/null +++ b/include/tkDNN/TrackingNN.h @@ -0,0 +1,158 @@ +#ifndef TRACKINGNN_H +#define TRACKINGNN_H + +#include +#include +#include +#ifdef __linux__ +#include +#endif + +#include +#include "utils.h" + +#include +#include +#include + +#include "tkdnn.h" + +// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib. + +#ifdef OPENCV_CUDACONTRIB +#include +#include +#endif + + +namespace tk { namespace dnn { + +class TrackingNN { + + protected: + tk::dnn::NetworkRT *netRT = nullptr; + dnnType *input_d; + + std::vector originalSize; + + cv::Scalar colors[256]; + + int nBatches = 1; + +#ifdef OPENCV_CUDACONTRIB + cv::cuda::GpuMat bgr[3]; + cv::cuda::GpuMat imagePreproc; +#else + cv::Mat bgr[3]; + cv::Mat imagePreproc; + dnnType *input; +#endif + + /** + * This method preprocess the image, before feeding it to the NN. + * + * @param frame original frame to adapt for inference. + * @param bi batch index + */ + virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; + + /** + * This method postprocess the output of the NN to obtain the correct + * boundig boxes. + * + * @param bi batch index + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation + */ + virtual void postprocess(const int bi=0,const bool mAP=false) = 0; + + public: + int classes = 0; + float confThreshold = 0.3; /*threshold on the confidence of the boxes*/ + + std::vector pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/ + std::vector classesNames; + + TrackingNN() {}; + ~TrackingNN(){}; + + /** + * Method used to initialize the class, allocate memory and compute + * needed data. + * + * @param tensor_path path to the rt file of the NN. + * @param n_classes number of classes for the given dataset. + * @param n_batches maximum number of batches to use in inference. + * @return true if everything is correct, false otherwise. + */ + virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, + const float conf_thresh=0.3, const bool mode_3d=true, const std::vector& k_calibs=std::vector()) = 0; + + /** + * This method performs the whole detection and tracking of the NN. + * + * @param frames frames to run detection and trcking on. + * @param cur_batches number of batches to use in inference. + * @param save_times if set to true, preprocess, inference and postprocess times + * are saved on a csv file, otherwise not. + * @param times pointer to the output stream where to write times. + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation. + */ + void update(std::vector& frames, const int cur_batches=1, bool save_times=false, + std::ofstream *times=nullptr, const bool mAP=false){ + if(save_times && times==nullptr) + FatalError("save_times set to true, but no valid ofstream given"); + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); + + originalSize.clear(); + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); + { + TKDNN_TSTART + for(int bi=0; biinput_dim; + dim.n = cur_batches; + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + if(save_times) *times<& frames){}; + +}; + +}} + +#endif /* TRACKINGNN_H*/ diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenterTrack.cpp similarity index 96% rename from src/CenternetDetection3DTrack.cpp rename to src/CenterTrack.cpp index d119c1e..dc15823 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenterTrack.cpp @@ -1,16 +1,17 @@ -#include "CenternetDetection3DTrack.h" +#include "CenterTrack.h" namespace tk { namespace dnn { -bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, - const float conf_thresh, const std::vector& k_calibs) { +bool CenterTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, + const float conf_thresh, const bool mode_3d, const std::vector& k_calibs) { netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); dim = netRT->input_dim; dim.c = 3; nBatches = n_batches; confThreshold = conf_thresh; + mode3D = mode_3d; inputCalibs = k_calibs; init_preprocessing(); init_pre_inf(); @@ -18,7 +19,7 @@ bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n init_visualization(n_classes); } -bool CenternetDetection3DTrack::init_preprocessing(){ +bool CenterTrack::init_preprocessing(){ //image transformation src = cv::Mat(cv::Size(2,3), CV_32F); dst = cv::Mat(cv::Size(2,3), CV_32F); @@ -60,12 +61,12 @@ bool CenternetDetection3DTrack::init_preprocessing(){ checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); } -bool CenternetDetection3DTrack::init_pre_inf(){ +bool CenterTrack::init_pre_inf(){ // initial steps: the first part of the network - const char *pre_img_conv1_bin = "dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; - const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; - const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; - const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; + const char *pre_img_conv1_bin = "dla34_ctrack/layers/base-pre_img_layer-0.bin"; + const char *pre_hm_conv1_bin = "dla34_ctrack/layers/base-pre_hm_layer-0.bin"; + const char *conv1_bin = "dla34_ctrack/layers/base-base_layer-0.bin"; + const char *conv2_bin = "dla34_ctrack/layers/base-level0-0.bin"; dim_in0 = tk::dnn::dataDim_t(1, 3, 512, 512, 1); dim_in1 = tk::dnn::dataDim_t(1, 1, 512, 512, 1); @@ -82,9 +83,9 @@ bool CenternetDetection3DTrack::init_pre_inf(){ dnnType *i0_h, *i1_h, *i2_h; // dnnType *i0_d, *i1_d, *i2_d; - // const char *input_bin = "dla34_cnet3d_track/debug/input.bin"; - // const char *pre_img_bin = "dla34_cnet3d_track/debug/pre_imgages.bin"; - // const char *pre_hm_bin = "dla34_cnet3d_track/debug/pre_hms.bin"; + // const char *input_bin = "dla34_ctrack/debug/input.bin"; + // const char *pre_img_bin = "dla34_ctrack/debug/pre_imgages.bin"; + // const char *pre_hm_bin = "dla34_ctrack/debug/pre_hms.bin"; // readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &img_d); // readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &hm_d); // readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &input_pre_inf_d); @@ -114,7 +115,7 @@ bool CenternetDetection3DTrack::init_pre_inf(){ return true; } -bool CenternetDetection3DTrack::init_postprocessing(){ +bool CenterTrack::init_postprocessing(){ srand(0); //seed = 0 for random colors dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1); @@ -203,7 +204,7 @@ bool CenternetDetection3DTrack::init_postprocessing(){ trackId.resize(nBatches, 0); } -bool CenternetDetection3DTrack::init_visualization(const int n_classes){ +bool CenterTrack::init_visualization(const int n_classes){ classes = n_classes; // const char *kitti_class_name[] = { // "person", "car", "bicycle"}; @@ -275,11 +276,11 @@ bool CenternetDetection3DTrack::init_visualization(const int n_classes){ // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } -void CenternetDetection3DTrack::_get_additional_inputs(){ +void CenterTrack::_get_additional_inputs(){ //None no additional input } -void CenternetDetection3DTrack::pre_inf(const int bi){ +void CenterTrack::pre_inf(const int bi){ TKDNN_TSTART tk::dnn::dataDim_t dim_aus; pre_phase_net->infer(dim_aus, nullptr); @@ -289,7 +290,7 @@ void CenternetDetection3DTrack::pre_inf(const int bi){ checkCuda( cudaDeviceSynchronize() ); } -void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ +void CenterTrack::preprocess(cv::Mat &frame, const int bi){ cv::Size sz = originalSize[bi]; // float scale = 1.0; float new_height = dim.h;//sz.height * scale; @@ -403,7 +404,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ checkCuda( cudaDeviceSynchronize() ); } -cv::Mat CenternetDetection3DTrack::transform_preds_with_trans(float x1, float x2){ +cv::Mat CenterTrack::transform_preds_with_trans(float x1, float x2){ cv::Mat target_coords(cv::Size(1,3), CV_32F); target_coords.at(0,0) = x1; target_coords.at(0,1) = x2; @@ -411,7 +412,7 @@ cv::Mat CenternetDetection3DTrack::transform_preds_with_trans(float x1, float x2 return transOut * target_coords; } -void CenternetDetection3DTrack::tracking(const int bi) { +void CenterTrack::tracking(const int bi) { float item_size[countDet]; int item_cl[countDet]; float dets[2*countDet]; @@ -600,7 +601,7 @@ void CenternetDetection3DTrack::tracking(const int bi) { } -void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { +void CenterTrack::postprocess(const int bi, const bool mAP) { dnnType *rt_out[9]; rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi; @@ -734,7 +735,7 @@ void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { tracking(bi); } -void CenternetDetection3DTrack::draw(std::vector& frames) { +void CenterTrack::draw(std::vector& frames) { struct trackingRes t; float sc; int id; @@ -755,7 +756,7 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { cv::Size text_size = getTextSize(txt, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); if(t.det_res.score > confThreshold){// && t.active!=0) { - if(view2d) { + if(!mode3D) { cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0) * scale_x, t.det_res.bb0.at(0,1) * scale_y), cv::Point(t.det_res.bb1.at(0,0) * scale_x, t.det_res.bb1.at(0,1) * scale_y), @@ -776,7 +777,7 @@ void CenternetDetection3DTrack::draw(std::vector& frames) { cv::Scalar(255, 0, 255), 2); } //3d - if(!view2d && t.det_res.z > 1){ + if(mode3D && t.det_res.z > 1){ r.at(0,0) = std::cos(t.det_res.rot_y); r.at(0,2) = std::sin(t.det_res.rot_y); r.at(2,0) = -std::sin(t.det_res.rot_y); diff --git a/tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp b/tests/centertrack/dla34_ctrack/dla34_ctrack.cpp similarity index 73% rename from tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp rename to tests/centertrack/dla34_ctrack/dla34_ctrack.cpp index 4829f16..eb3788c 100644 --- a/tests/centernet/dla34_cnet3d_track/dla34_cnet3d_track.cpp +++ b/tests/centertrack/dla34_ctrack/dla34_ctrack.cpp @@ -1,130 +1,130 @@ #include #include "tkdnn.h" -const char *input_bin = "dla34_cnet3d_track/debug/input_base-level0-0.bin"; -// const char *input_bin = "dla34_cnet3d_track/debug/input.bin"; -// const char *pre_img_bin = "dla34_cnet3d_track/debug/pre_imgages.bin"; -// const char *pre_hm_bin = "dla34_cnet3d_track/debug/pre_hms.bin"; +const char *input_bin = "dla34_ctrack/debug/input_base-level0-0.bin"; +// const char *input_bin = "dla34_ctrack/debug/input.bin"; +// const char *pre_img_bin = "dla34_ctrack/debug/pre_imgages.bin"; +// const char *pre_hm_bin = "dla34_ctrack/debug/pre_hms.bin"; // //pre -// const char *pre_img_conv1_bin = "dla34_cnet3d_track/layers/base-pre_img_layer-0.bin"; -// const char *pre_hm_conv1_bin = "dla34_cnet3d_track/layers/base-pre_hm_layer-0.bin"; -// const char *conv1_bin = "dla34_cnet3d_track/layers/base-base_layer-0.bin"; +// const char *pre_img_conv1_bin = "dla34_ctrack/layers/base-pre_img_layer-0.bin"; +// const char *pre_hm_conv1_bin = "dla34_ctrack/layers/base-pre_hm_layer-0.bin"; +// const char *conv1_bin = "dla34_ctrack/layers/base-base_layer-0.bin"; -const char *conv2_bin = "dla34_cnet3d_track/layers/base-level0-0.bin"; -const char *conv3_bin = "dla34_cnet3d_track/layers/base-level1-0.bin"; +const char *conv2_bin = "dla34_ctrack/layers/base-level0-0.bin"; +const char *conv3_bin = "dla34_ctrack/layers/base-level1-0.bin"; // s - stage, t - tree -const char *s1_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level2-tree1-conv1.bin"; -const char *s1_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level2-tree1-conv2.bin"; -const char *s1_t1_project = "dla34_cnet3d_track/layers/base-level2-project-0.bin"; -const char *s1_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level2-tree2-conv1.bin"; -const char *s1_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level2-tree2-conv2.bin"; -const char *s1_root_conv1_bin = "dla34_cnet3d_track/layers/base-level2-root-conv.bin"; -const char *s2_t1_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree1-conv1.bin"; -const char *s2_t1_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree1-conv2.bin"; -const char *s2_t1_t1_project = "dla34_cnet3d_track/layers/base-level3-tree1-project-0.bin"; -const char *s2_t1_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree2-conv1.bin"; -const char *s2_t1_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree1-tree2-conv2.bin"; -const char *s2_t1_root_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree1-root-conv.bin"; -const char *s2_t2_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree1-conv1.bin"; -const char *s2_t2_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree1-conv2.bin"; -const char *s2_t2_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree2-conv1.bin"; -const char *s2_t2_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level3-tree2-tree2-conv2.bin"; -const char *s2_t2_root_conv1_bin = "dla34_cnet3d_track/layers/base-level3-tree2-root-conv.bin"; -const char *s3_t1_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree1-conv1.bin"; -const char *s3_t1_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree1-conv2.bin"; -const char *s3_t1_t1_project = "dla34_cnet3d_track/layers/base-level4-tree1-project-0.bin"; -const char *s3_t1_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree2-conv1.bin"; -const char *s3_t1_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree1-tree2-conv2.bin"; -const char *s3_t1_root_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree1-root-conv.bin"; -const char *s3_t2_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree1-conv1.bin"; -const char *s3_t2_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree1-conv2.bin"; -const char *s3_t2_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree2-conv1.bin"; -const char *s3_t2_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level4-tree2-tree2-conv2.bin"; -const char *s3_t2_root_conv1_bin = "dla34_cnet3d_track/layers/base-level4-tree2-root-conv.bin"; -const char *s4_t1_conv1_bin = "dla34_cnet3d_track/layers/base-level5-tree1-conv1.bin"; -const char *s4_t1_conv2_bin = "dla34_cnet3d_track/layers/base-level5-tree1-conv2.bin"; -const char *s4_t1_project = "dla34_cnet3d_track/layers/base-level5-project-0.bin"; -const char *s4_t2_conv1_bin = "dla34_cnet3d_track/layers/base-level5-tree2-conv1.bin"; -const char *s4_t2_conv2_bin = "dla34_cnet3d_track/layers/base-level5-tree2-conv2.bin"; -const char *s4_root_conv1_bin = "dla34_cnet3d_track/layers/base-level5-root-conv.bin"; +const char *s1_t1_conv1_bin = "dla34_ctrack/layers/base-level2-tree1-conv1.bin"; +const char *s1_t1_conv2_bin = "dla34_ctrack/layers/base-level2-tree1-conv2.bin"; +const char *s1_t1_project = "dla34_ctrack/layers/base-level2-project-0.bin"; +const char *s1_t2_conv1_bin = "dla34_ctrack/layers/base-level2-tree2-conv1.bin"; +const char *s1_t2_conv2_bin = "dla34_ctrack/layers/base-level2-tree2-conv2.bin"; +const char *s1_root_conv1_bin = "dla34_ctrack/layers/base-level2-root-conv.bin"; +const char *s2_t1_t1_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-tree1-conv1.bin"; +const char *s2_t1_t1_conv2_bin = "dla34_ctrack/layers/base-level3-tree1-tree1-conv2.bin"; +const char *s2_t1_t1_project = "dla34_ctrack/layers/base-level3-tree1-project-0.bin"; +const char *s2_t1_t2_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-tree2-conv1.bin"; +const char *s2_t1_t2_conv2_bin = "dla34_ctrack/layers/base-level3-tree1-tree2-conv2.bin"; +const char *s2_t1_root_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-root-conv.bin"; +const char *s2_t2_t1_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-tree1-conv1.bin"; +const char *s2_t2_t1_conv2_bin = "dla34_ctrack/layers/base-level3-tree2-tree1-conv2.bin"; +const char *s2_t2_t2_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-tree2-conv1.bin"; +const char *s2_t2_t2_conv2_bin = "dla34_ctrack/layers/base-level3-tree2-tree2-conv2.bin"; +const char *s2_t2_root_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-root-conv.bin"; +const char *s3_t1_t1_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-tree1-conv1.bin"; +const char *s3_t1_t1_conv2_bin = "dla34_ctrack/layers/base-level4-tree1-tree1-conv2.bin"; +const char *s3_t1_t1_project = "dla34_ctrack/layers/base-level4-tree1-project-0.bin"; +const char *s3_t1_t2_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-tree2-conv1.bin"; +const char *s3_t1_t2_conv2_bin = "dla34_ctrack/layers/base-level4-tree1-tree2-conv2.bin"; +const char *s3_t1_root_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-root-conv.bin"; +const char *s3_t2_t1_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-tree1-conv1.bin"; +const char *s3_t2_t1_conv2_bin = "dla34_ctrack/layers/base-level4-tree2-tree1-conv2.bin"; +const char *s3_t2_t2_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-tree2-conv1.bin"; +const char *s3_t2_t2_conv2_bin = "dla34_ctrack/layers/base-level4-tree2-tree2-conv2.bin"; +const char *s3_t2_root_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-root-conv.bin"; +const char *s4_t1_conv1_bin = "dla34_ctrack/layers/base-level5-tree1-conv1.bin"; +const char *s4_t1_conv2_bin = "dla34_ctrack/layers/base-level5-tree1-conv2.bin"; +const char *s4_t1_project = "dla34_ctrack/layers/base-level5-project-0.bin"; +const char *s4_t2_conv1_bin = "dla34_ctrack/layers/base-level5-tree2-conv1.bin"; +const char *s4_t2_conv2_bin = "dla34_ctrack/layers/base-level5-tree2-conv2.bin"; +const char *s4_root_conv1_bin = "dla34_ctrack/layers/base-level5-root-conv.bin"; //final -// const char *fc_bin = "dla34_cnet3d_track/layers/output.bin"; +// const char *fc_bin = "dla34_ctrack/layers/output.bin"; -const char *ida_0_p_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-proj_1-conv.bin"; -const char *ida_0_p_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin"; -const char *ida_0_up_1_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-up_1.bin"; -const char *ida_0_n_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-node_1-conv.bin"; -const char *ida_0_n_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin"; +const char *ida_0_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_0-proj_1-conv.bin"; +const char *ida_0_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin"; +const char *ida_0_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_0-up_1.bin"; +const char *ida_0_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_0-node_1-conv.bin"; +const char *ida_0_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin"; -const char *ida_1_p_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_1-conv.bin"; -const char *ida_1_p_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin"; -const char *ida_1_up_1_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-up_1.bin"; -const char *ida_1_n_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_1-conv.bin"; -const char *ida_1_n_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin"; -const char *ida_1_p_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_2-conv.bin"; -const char *ida_1_p_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin"; -const char *ida_1_up_2_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-up_2.bin"; -const char *ida_1_n_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_2-conv.bin"; -const char *ida_1_n_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin"; +const char *ida_1_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_1-conv.bin"; +const char *ida_1_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin"; +const char *ida_1_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_1-up_1.bin"; +const char *ida_1_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-node_1-conv.bin"; +const char *ida_1_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin"; +const char *ida_1_p_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_2-conv.bin"; +const char *ida_1_p_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin"; +const char *ida_1_up_2_deconv_bin = "dla34_ctrack/layers/dla_up-ida_1-up_2.bin"; +const char *ida_1_n_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-node_2-conv.bin"; +const char *ida_1_n_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin"; -const char *ida_2_p_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_1-conv.bin"; -const char *ida_2_p_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin"; -const char *ida_2_up_1_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-up_1.bin"; -const char *ida_2_n_1_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_1-conv.bin"; -const char *ida_2_n_1_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin"; -const char *ida_2_p_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_2-conv.bin"; -const char *ida_2_p_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin"; -const char *ida_2_up_2_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-up_2.bin"; -const char *ida_2_n_2_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_2-conv.bin"; -const char *ida_2_n_2_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin"; -const char *ida_2_p_3_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_3-conv.bin"; -const char *ida_2_p_3_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin"; -const char *ida_2_up_3_deconv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-up_3.bin"; -const char *ida_2_n_3_dcn_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_3-conv.bin"; -const char *ida_2_n_3_conv_bin = "dla34_cnet3d_track/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin"; +const char *ida_2_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_1-conv.bin"; +const char *ida_2_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin"; +const char *ida_2_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_1.bin"; +const char *ida_2_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_1-conv.bin"; +const char *ida_2_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin"; +const char *ida_2_p_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_2-conv.bin"; +const char *ida_2_p_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin"; +const char *ida_2_up_2_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_2.bin"; +const char *ida_2_n_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_2-conv.bin"; +const char *ida_2_n_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin"; +const char *ida_2_p_3_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_3-conv.bin"; +const char *ida_2_p_3_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin"; +const char *ida_2_up_3_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_3.bin"; +const char *ida_2_n_3_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_3-conv.bin"; +const char *ida_2_n_3_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin"; -const char *ida_up_p_1_dcn_bin = "dla34_cnet3d_track/layers/ida_up-proj_1-conv.bin"; -const char *ida_up_p_1_conv_bin = "dla34_cnet3d_track/layers/ida_up-proj_1-conv-conv_offset_mask.bin"; -const char *ida_up_up_1_deconv_bin = "dla34_cnet3d_track/layers/ida_up-up_1.bin"; -const char *ida_up_n_1_dcn_bin = "dla34_cnet3d_track/layers/ida_up-node_1-conv.bin"; -const char *ida_up_n_1_conv_bin = "dla34_cnet3d_track/layers/ida_up-node_1-conv-conv_offset_mask.bin"; -const char *ida_up_p_2_dcn_bin = "dla34_cnet3d_track/layers/ida_up-proj_2-conv.bin"; -const char *ida_up_p_2_conv_bin = "dla34_cnet3d_track/layers/ida_up-proj_2-conv-conv_offset_mask.bin"; -const char *ida_up_up_2_deconv_bin = "dla34_cnet3d_track/layers/ida_up-up_2.bin"; -const char *ida_up_n_2_dcn_bin = "dla34_cnet3d_track/layers/ida_up-node_2-conv.bin"; -const char *ida_up_n_2_conv_bin = "dla34_cnet3d_track/layers/ida_up-node_2-conv-conv_offset_mask.bin"; +const char *ida_up_p_1_dcn_bin = "dla34_ctrack/layers/ida_up-proj_1-conv.bin"; +const char *ida_up_p_1_conv_bin = "dla34_ctrack/layers/ida_up-proj_1-conv-conv_offset_mask.bin"; +const char *ida_up_up_1_deconv_bin = "dla34_ctrack/layers/ida_up-up_1.bin"; +const char *ida_up_n_1_dcn_bin = "dla34_ctrack/layers/ida_up-node_1-conv.bin"; +const char *ida_up_n_1_conv_bin = "dla34_ctrack/layers/ida_up-node_1-conv-conv_offset_mask.bin"; +const char *ida_up_p_2_dcn_bin = "dla34_ctrack/layers/ida_up-proj_2-conv.bin"; +const char *ida_up_p_2_conv_bin = "dla34_ctrack/layers/ida_up-proj_2-conv-conv_offset_mask.bin"; +const char *ida_up_up_2_deconv_bin = "dla34_ctrack/layers/ida_up-up_2.bin"; +const char *ida_up_n_2_dcn_bin = "dla34_ctrack/layers/ida_up-node_2-conv.bin"; +const char *ida_up_n_2_conv_bin = "dla34_ctrack/layers/ida_up-node_2-conv-conv_offset_mask.bin"; -const char *hm_conv1_bin = "dla34_cnet3d_track/layers/hm-0.bin"; -const char *hm_conv2_bin = "dla34_cnet3d_track/layers/hm-2.bin"; -const char *wh_conv1_bin = "dla34_cnet3d_track/layers/wh-0.bin"; -const char *wh_conv2_bin = "dla34_cnet3d_track/layers/wh-2.bin"; -const char *reg_conv1_bin = "dla34_cnet3d_track/layers/reg-0.bin"; -const char *reg_conv2_bin = "dla34_cnet3d_track/layers/reg-2.bin"; -const char *track_conv1_bin = "dla34_cnet3d_track/layers/tracking-0.bin"; -const char *track_conv2_bin = "dla34_cnet3d_track/layers/tracking-2.bin"; -const char *dep_conv1_bin = "dla34_cnet3d_track/layers/dep-0.bin"; -const char *dep_conv2_bin = "dla34_cnet3d_track/layers/dep-2.bin"; -const char *rot_conv1_bin = "dla34_cnet3d_track/layers/rot-0.bin"; -const char *rot_conv2_bin = "dla34_cnet3d_track/layers/rot-2.bin"; -const char *dim_conv1_bin = "dla34_cnet3d_track/layers/dim-0.bin"; -const char *dim_conv2_bin = "dla34_cnet3d_track/layers/dim-2.bin"; -const char *a_off_conv1_bin = "dla34_cnet3d_track/layers/amodel_offset-0.bin"; -const char *a_off_conv2_bin = "dla34_cnet3d_track/layers/amodel_offset-2.bin"; +const char *hm_conv1_bin = "dla34_ctrack/layers/hm-0.bin"; +const char *hm_conv2_bin = "dla34_ctrack/layers/hm-2.bin"; +const char *wh_conv1_bin = "dla34_ctrack/layers/wh-0.bin"; +const char *wh_conv2_bin = "dla34_ctrack/layers/wh-2.bin"; +const char *reg_conv1_bin = "dla34_ctrack/layers/reg-0.bin"; +const char *reg_conv2_bin = "dla34_ctrack/layers/reg-2.bin"; +const char *track_conv1_bin = "dla34_ctrack/layers/tracking-0.bin"; +const char *track_conv2_bin = "dla34_ctrack/layers/tracking-2.bin"; +const char *dep_conv1_bin = "dla34_ctrack/layers/dep-0.bin"; +const char *dep_conv2_bin = "dla34_ctrack/layers/dep-2.bin"; +const char *rot_conv1_bin = "dla34_ctrack/layers/rot-0.bin"; +const char *rot_conv2_bin = "dla34_ctrack/layers/rot-2.bin"; +const char *dim_conv1_bin = "dla34_ctrack/layers/dim-0.bin"; +const char *dim_conv2_bin = "dla34_ctrack/layers/dim-2.bin"; +const char *a_off_conv1_bin = "dla34_ctrack/layers/amodel_offset-0.bin"; +const char *a_off_conv2_bin = "dla34_ctrack/layers/amodel_offset-2.bin"; const char *output_bin[]={ -"dla34_cnet3d_track/debug/hm.bin", -"dla34_cnet3d_track/debug/wh.bin", -"dla34_cnet3d_track/debug/reg.bin", -"dla34_cnet3d_track/debug/tracking.bin", -"dla34_cnet3d_track/debug/dep.bin", -"dla34_cnet3d_track/debug/rot.bin", -"dla34_cnet3d_track/debug/dim.bin", -"dla34_cnet3d_track/debug/amodel_offset.bin"}; -// const char *output_bin = "dla34_cnet3d_track/debug/base-level0-2.bin"; +"dla34_ctrack/debug/hm.bin", +"dla34_ctrack/debug/wh.bin", +"dla34_ctrack/debug/reg.bin", +"dla34_ctrack/debug/tracking.bin", +"dla34_ctrack/debug/dep.bin", +"dla34_ctrack/debug/rot.bin", +"dla34_ctrack/debug/dim.bin", +"dla34_ctrack/debug/amodel_offset.bin"}; +// const char *output_bin = "dla34_ctrack/debug/base-level0-2.bin"; int main() { - downloadWeightsifDoNotExist("dla34_cnet3d_track/debug/input.bin", "dla34_cnet3d_track", "https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download"); + downloadWeightsifDoNotExist("dla34_ctrack/debug/input.bin", "dla34_ctrack", "https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download"); // Network layout // tk::dnn::dataDim_t dim_in0(1, 3, 512, 512, 1); @@ -570,7 +570,7 @@ int main() net.print(); //convert network to tensorRT - tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet3d_track")); + tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_ctrack")); tk::dnn::dataDim_t dim1 = dim_in0; //input dim printCenteredTitle(" CUDNN inference ", '=', 30); -- 2.52.0 From fd56e64938d2427de88854f519d628f0bc4ae899 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 11 May 2021 17:02:43 +0200 Subject: [PATCH 132/228] Update the README and split it into several files. Signed-off-by: Davide Sapienza --- README.md | 339 +------------------------------------- docs/demo.md | 213 ++++++++++++++++++++++++ docs/exporting_weights.md | 100 +++++++++++ docs/mAP_demo.md | 34 ++++ docs/windows.md | 95 +++++++++++ 5 files changed, 447 insertions(+), 334 deletions(-) create mode 100644 docs/demo.md create mode 100644 docs/exporting_weights.md create mode 100644 docs/mAP_demo.md create mode 100644 docs/windows.md diff --git a/README.md b/README.md index 9e4b811..12f2104 100644 --- a/README.md +++ b/README.md @@ -70,26 +70,11 @@ Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001 - [How to compile this repo](#how-to-compile-this-repo) - [Workflow](#workflow) - [How to export weights](#how-to-export-weights) - - [1)Export weights from darknet](#1export-weights-from-darknet) - - [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101) - - [3)Export weights for CenterNet](#3export-weights-for-centernet) - - [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd) - [Run the demo](#run-the-demo) - - [FP16 inference](#fp16-inference) - - [INT8 inference](#int8-inference) - [mAP demo](#map-demo) - [Existing tests and supported networks](#existing-tests-and-supported-networks) - [References](#references) - [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows-10-experimental) - - [Dependencies-Windows](#dependencies-windows) - - [Compiling tkDNN on Windows](#compiling-tkdnn-on-windows) - - [Run the demo on Windows](#run-the-demo-on-windows) - - [FP16 inference windows](#fp16-inference-windows) - - [INT8 inference windows](#int8-inference-windows) - - [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows) - - - ## Dependencies @@ -126,246 +111,17 @@ Steps needed to do inference on tkDNN with a custom neural network. * Create a new test and define the network, layer by layer using the weights extracted and the output to check the results. * Do inference. -## How to export weights +## Exporting weights -Weights are essential for any network to run inference. For each test a folder organized as follow is needed (in the build folder): -``` - test_nn - |---- layers/ (folder containing a binary file for each layer with the corresponding wieghts and bias) - |---- debug/ (folder containing a binary file for each layer with the corresponding outputs) -``` -Therefore, once the weights have been exported, the folders layers and debug should be placed in the corresponding test. - -### 1)Export weights from darknet -To export weights for NNs that are defined in darknet framework, use [this](https://git.hipert.unimore.it/fgatti/darknet.git) fork of darknet and follow these steps to obtain a correct debug and layers folder, ready for tkDNN. - -``` -git clone https://git.hipert.unimore.it/fgatti/darknet.git -cd darknet -make -mkdir layers debug -./darknet export layers -``` -N.b. Use compilation with CPU (leave GPU=0 in Makefile) if you also want debug. - -### 2)Export weights for DLA34 and ResNet101 -To get weights and outputs needed to run the tests dla34 and resnet101 use the Python script and the Anaconda environment included in the repository. - -Create Anaconda environment and activate it: -``` -conda env create -f file_name.yml -source activate env_name -python