From ae1d8cd9e60f91e4927d6025c73fe6bea63851c0 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Wed, 8 Apr 2020 11:20:48 +0200 Subject: [PATCH] Refactoring & documentation Signed-off-by: Micaela Verucchi --- demo/demo/map.cpp | 2 +- include/tkDNN/DetectionNN.h | 21 +++---- include/tkDNN/evaluation.h | 96 ++++++++++++++++++++++------- src/CenternetDetection.cpp | 9 +-- src/MobilenetDetection.cpp | 18 ++---- src/Reshape.cpp | 5 +- src/Yolo3Detection.cpp | 9 +-- src/evaluation.cpp | 117 +++++++++--------------------------- src/utils.cpp | 24 +++----- 9 files changed, 133 insertions(+), 168 deletions(-) diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index db98726..b380c5d 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -34,7 +34,7 @@ int main(int argc, char *argv[]) bool show = false; bool write_dets = false; bool write_res_on_file = true; - int n_images = 50; + int n_images = 5000; bool verbose; int classes, map_points, map_levels; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index d884faf..ccb379c 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -24,12 +24,6 @@ namespace tk { namespace dnn { -enum networkType_t{ - NETWORK_YOLO3, - NETWORK_MOBILENETSSDLITE, - NETWORK_CENTERNET -}; - class DetectionNN { protected: @@ -52,7 +46,7 @@ class DetectionNN { /** * This method preprocess the image, before feeding it to the NN. * - * @param original frame to adapt for inference. + * @param frame original frame to adapt for inference. */ virtual void preprocess(cv::Mat &frame) = 0; @@ -78,7 +72,8 @@ class DetectionNN { * Method used to inialize the class, allocate memory and compute * needed data. * - * @param path to the rt file og the NN. + * @param tensor_path path to the rt file og the NN. + * @param n_classes number of classes for the given dataset. * @return true if everything is correct, false otherwise. */ virtual bool init(const std::string& tensor_path, const int n_classes=80) = 0; @@ -86,9 +81,10 @@ class DetectionNN { /** * This method performs the whole detection of the NN. * - * @param frame to run detection on. - * @param if set to true, preprocess, inference and postprocess times + * @param frame frame to run detection on. + * @param save_times if set to true, preprocess, inference and postprocess times * are saved on a csv file, otherwise not. + * @param times pointer to the output stream where to write times */ void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){ if(!frame.data) @@ -129,11 +125,10 @@ class DetectionNN { /** * Method to draw boundixg boxes and labels on a frame. * - * @param orginal frame to draw bounding box on. + * @param frame orginal frame to draw bounding box on. * @return frame with boundig boxes. */ - cv::Mat draw(cv::Mat &frame) - { + cv::Mat draw(cv::Mat &frame) { tk::dnn::box b; int x0, w, x1, y0, h, y1; int objClass; diff --git a/include/tkDNN/evaluation.h b/include/tkDNN/evaluation.h index 98a4166..b8d68d5 100644 --- a/include/tkDNN/evaluation.h +++ b/include/tkDNN/evaluation.h @@ -8,20 +8,9 @@ #include #include "tkdnn.h" +#include "BoundingBox.h" namespace tk { namespace dnn { -struct BoundingBox : public tk::dnn::box -{ - friend std::ostream& operator<<(std::ostream& os, const BoundingBox& bb); - int uniqueTruthIndex = -1; - int truthFlag = 0; - float maxIoU = 0; - - void clear(); -}; - -std::ostream& operator<<(std::ostream& os, const BoundingBox& bb); -bool boxComparison (const BoundingBox& a,const BoundingBox& b) ; struct Frame { @@ -42,19 +31,82 @@ struct PR void print(); }; -float boxOverlap(float x1, float w1, float x2, float w2); -float boxIntersection(const BoundingBox &a, const BoundingBox &b); -float boxUnion(const BoundingBox &a, const BoundingBox &b); -float boxIoU(const BoundingBox &a, const BoundingBox &b); +void readmAPParams( char* config_filename, int& classes, int& map_points, + int& map_levels, float& map_step, float& IoU_thresh, + float& conf_thresh, bool& verbose); -void readmAPParams(char* config_filename, int& classes, int& map_points, - int& map_levels, float& map_step, float& IoU_thresh, - float& conf_thresh, bool& verbose); +/** + * This method computes the mean Average Precision for a set of detections and + * groundtruths. It returns the mAP for a given IoU threshold, and a given + * confidence threshold over all the classes. + * + * @param images collection of frames on which to compute the metrics + * @param classes number of classes of the considered dataset + * @param IoU_thresh threshold used to compute Intersection over Union + * @param conf_thresh threshold used to filter bounding boxes based on their + * confidence (or probability) + * @param map_points number of point used to compute the mAP. if 0 is given, + * all the recall levels are evaluated, otherwise only + * map_point recall levels are used. For COCO evaluation + * 101 points are used. + * @param verbose is set to true, prints on screen additional info + * + * @return mAP computed + */ +double computeMap( std::vector &images,const int classes, + const float IoU_thresh, const float conf_thresh=0.3, + const int map_points=101, const bool verbose=false); -double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh=0.3, const int map_points=101, const bool verbose=false); -double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const float conf_thresh=0.3, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false, const bool write_on_file = false, std::string net = ""); -void computeTPFPFN(std::vector &images,const int classes,const float IoU_thresh=0.5, const float conf_thresh=0.3, bool verbose=false, const bool write_on_file=false, std::string net=""); +/** + * This method computes the mean Average Precision for a set of detections and + * groundtruths on several IoU thresholds. It is used to compute, for example, + * the most used metric in Object Detection, namely the mAP 0.5:0.95, which is + * the average among the mAP for IoU level from 0.5 to 0.95 with a step of 0.05. + * + * @param images collection of frames on which to compute the metrics + * @param classes number of classes of the considered dataset + * @param IoU_thresh starting threshold used to compute Intersection over Union + * @param conf_thresh threshold used to filter bounding boxes based on their + * confidence (or probability) + * @param map_points number of point used to compute the mAP. if 0 is given, + * all the recall levels are evaluated, otherwise only + * map_point recall levels are used. For COCO evaluation + * 101 points are used. + * @param map_step step used to increment IoU theshold + * @param map_levels number of IoU step to perform + * @param verbose is set to true, prints on screen additional info + * @param write_on_file if set to true, the results produced by this function + * are written on file + * @param net name of the considerd neural network + * + * @return mAP IoU_tresh:IoU_tresh+map_step*map_levels (e.g. mAP 0.5:0.95 when + * map_step=0.05 and map_levels=10) + */ +double computeMapNIoULevels(std::vector &images,const int classes, + const float i_IoU_thresh=0.5, const float conf_thresh=0.3, + const int map_points=101, const float map_step=0.05, + const int map_levels=10, const bool verbose=false, + const bool write_on_file = false, std::string net = ""); +/** + * This method computes the numper of True Positive (TP), False Positive (FP), + * False Negative (FN), precision, recall and f1-score. + * Those values are computer over all the detections, over all the classes. + * + * @param images collection of frames on which to compute the metrics + * @param classes number of classes of the considered dataset + * @param IoU_thresh threshold used to compute Intersection over Union + * @param conf_thresh threshold used to filter bounding boxes based on their + * confidence (or probability) + * @param verbose is set to true, prints on screen additional info + * @param write_on_file if set to true, the results produced by this function + * are written on file + * @param net name of the considerd neural network + */ +void computeTPFPFN( std::vector &images,const int classes, + const float IoU_thresh=0.5, const float conf_thresh=0.3, + bool verbose=false, const bool write_on_file=false, + std::string net=""); }} #endif /*EVALUATION_H*/ diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 248ba55..997989e 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -3,8 +3,7 @@ namespace tk { namespace dnn { -bool CenternetDetection::init(const std::string& tensor_path, const int n_classes) -{ +bool CenternetDetection::init(const std::string& tensor_path, const int n_classes){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; @@ -121,8 +120,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe } -void CenternetDetection::preprocess(cv::Mat &frame) -{ +void CenternetDetection::preprocess(cv::Mat &frame){ // -----------------------------------pre-process ------------------------------------------ // auto start_t = std::chrono::steady_clock::now(); @@ -262,8 +260,7 @@ void CenternetDetection::preprocess(cv::Mat &frame) #endif } -void CenternetDetection::postprocess() -{ +void CenternetDetection::postprocess(){ dnnType *rt_out[4]; rt_out[0] = (dnnType *)netRT->buffersRT[1]; rt_out[1] = (dnnType *)netRT->buffersRT[2]; diff --git a/src/MobilenetDetection.cpp b/src/MobilenetDetection.cpp index 4c6ff8a..3f27de1 100644 --- a/src/MobilenetDetection.cpp +++ b/src/MobilenetDetection.cpp @@ -6,8 +6,7 @@ bool boxProbCmp(const tk::dnn::box &a, const tk::dnn::box &b){ namespace tk{ namespace dnn{ -void MobilenetDetection::generate_ssd_priors(const SSDSpec *specs, const int n_specs, bool clamp) -{ +void MobilenetDetection::generate_ssd_priors(const SSDSpec *specs, const int n_specs, bool clamp){ nPriors = 0; for (int i = 0; i < n_specs; i++){ nPriors += specs[i].featureSize * specs[i].featureSize * 6; @@ -86,8 +85,7 @@ void MobilenetDetection::generate_ssd_priors(const SSDSpec *specs, const int n_s } } -void MobilenetDetection::convert_locatios_to_boxes_and_center() -{ +void MobilenetDetection::convert_locatios_to_boxes_and_center(){ float cur_x, cur_y; for (int i = 0; i < nPriors; i++){ locations_h[i * N_COORDS + 0] = locations_h[i * N_COORDS + 0] * centerVariance * priors[i * N_COORDS + 2] + priors[i * N_COORDS + 0]; @@ -105,8 +103,7 @@ void MobilenetDetection::convert_locatios_to_boxes_and_center() } } -float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b) -{ +float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){ float max_x = a.x > b.x ? a.x : b.x; float max_y = a.y > b.y ? a.y : b.y; float min_w = a.w < b.w ? a.w : b.w; @@ -129,8 +126,7 @@ float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b) return iou; } -bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes) -{ +bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); imageSize = netRT->input_dim.h; @@ -207,8 +203,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe return 1; } -void MobilenetDetection::preprocess(cv::Mat &frame) -{ +void MobilenetDetection::preprocess(cv::Mat &frame){ #ifdef OPENCV_CUDACONTRIB //move original image on GPU cv::cuda::GpuMat orig_img, frame_nomean; @@ -243,8 +238,7 @@ void MobilenetDetection::preprocess(cv::Mat &frame) #endif } -void MobilenetDetection::postprocess() -{ +void MobilenetDetection::postprocess(){ //get confidences and locations_h dnnType *rt_out[2]; rt_out[0] = (dnnType *)netRT->buffersRT[3]; diff --git a/src/Reshape.cpp b/src/Reshape.cpp index 4e83cf1..1a3a8f9 100644 --- a/src/Reshape.cpp +++ b/src/Reshape.cpp @@ -24,11 +24,8 @@ Reshape::~Reshape() { dnnType* Reshape::infer(dataDim_t &dim, dnnType* srcData) { - //transpose per channel - + //just copies the data and changes the output dim checkCuda( cudaMemcpy(dstData, srcData, dim.n*dim.c*dim.h*dim.w*sizeof(dnnType), cudaMemcpyDeviceToDevice)); - - //update data dimensions dim = output_dim; return dstData; diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index 654b5db..9cd57c3 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -48,8 +48,7 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes) { return true; } -void Yolo3Detection::preprocess(cv::Mat &frame) -{ +void Yolo3Detection::preprocess(cv::Mat &frame){ #ifdef OPENCV_CUDACONTRIB cv::cuda::GpuMat orig_img, img_resized; orig_img = cv::cuda::GpuMat(frame); @@ -84,8 +83,7 @@ void Yolo3Detection::preprocess(cv::Mat &frame) #endif } -void Yolo3Detection::postprocess() -{ +void Yolo3Detection::postprocess(){ //get yolo outputs dnnType *rt_out[netRT->pluginFactory->n_yolos]; for(int i=0; ipluginFactory->n_yolos; i++) { @@ -140,8 +138,7 @@ void Yolo3Detection::postprocess() } -tk::dnn::Yolo* Yolo3Detection::getYoloLayer(int n) -{ +tk::dnn::Yolo* Yolo3Detection::getYoloLayer(int n) { if(n<3) return yolo[n]; else diff --git a/src/evaluation.cpp b/src/evaluation.cpp index b41d88d..cc0118f 100644 --- a/src/evaluation.cpp +++ b/src/evaluation.cpp @@ -3,29 +3,7 @@ namespace tk { namespace dnn { -void BoundingBox::clear() -{ - uniqueTruthIndex = -1; - truthFlag = 0; - maxIoU = 0; -} - -bool boxComparison (const BoundingBox& a,const BoundingBox& b) -{ - return (a.prob>b.prob); -} - -std::ostream& operator<<(std::ostream& os, const BoundingBox& bb) -{ - os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << - ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< - bb.truthFlag<< ", assignedGT: "<< bb.uniqueTruthIndex<< - ", maxIoU: "<< bb.maxIoU<<"\n"; - return os; -} - -void Frame::print() const -{ +void Frame::print() const{ std::cout<<"labels filename: "< l2 ? l1 : l2; - float r1 = x1 + w1/2; - float r2 = x2 + w2/2; - float right = r1 < r2 ? r1 : r2; - return right - left; -} - -float boxIntersection(const BoundingBox &a, const BoundingBox &b) -{ - float w = boxOverlap(a.x, a.w, b.x, b.w); - float h = boxOverlap(a.y, a.h, b.y, b.h); - if(w < 0 || h < 0) - return 0; - float area = w*h; - return area; -} - -float boxUnion(const BoundingBox &a, const BoundingBox &b) -{ - float i = boxIntersection(a, b); - float u = a.w*a.h + b.w*b.h - i; - return u; -} - -float boxIoU(const BoundingBox &a, const BoundingBox &b) -{ - float I = boxIntersection(a, b); - float U = boxUnion(a, b); - if (I == 0 || U == 0) - return 0; - return I / U; -} - -void readmAPParams(char* config_filename, int& classes, int& map_points, - int& map_levels, float& map_step, float& IoU_thresh, - float& conf_thresh, bool& verbose) -{ +void readmAPParams( char* config_filename, int& classes, int& map_points, + int& map_levels, float& map_step, float& IoU_thresh, + float& conf_thresh, bool& verbose) { YAML::Node config = YAML::LoadFile(config_filename); classes = config["classes"].as(); map_points = config["map_points"].as(); @@ -88,12 +27,12 @@ void readmAPParams(char* config_filename, int& classes, int& map_points, IoU_thresh = config["IoU_thresh"].as(); conf_thresh = config["conf_thresh"].as(); verbose = config["verbose"].as(); - } /* Credits to https://github.com/AlexeyAB/darknet/blob/master/src/detector.c*/ -double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh, const int map_points, const bool verbose) -{ +double computeMap( std::vector &images,const int classes, + const float IoU_thresh, const float conf_thresh, + const int map_points, const bool verbose) { if(verbose) for(auto img:images) img.print(); @@ -132,15 +71,13 @@ double computeMap(std::vector &images,const int classes,const float IoU_t float maxIoU = 0; int truth_index = -1; for(size_t j=0; j maxIoU && img.det[i].cl == img.gt[j].cl){ maxIoU = currentIoU; truth_index = j; } } - // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh){ - // std::cout<<"(INSIDE) IoU thresh:"< IoU_thresh:"<<(maxIoU > IoU_thresh)< &images,const int classes,const float IoU_t return mean_average_precision; } -double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh, const float conf_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose, const bool write_on_file, std::string net) -{ +double computeMapNIoULevels(std::vector &images,const int classes, + const float i_IoU_thresh, const float conf_thresh, + const int map_points, const float map_step, + const int map_levels, const bool verbose, + const bool write_on_file, std::string net) { std::ofstream out_file; if(write_on_file){ out_file.open("map.csv", std::ios_base::app); @@ -272,15 +212,18 @@ double computeMapNIoULevels(std::vector &images,const int classes,const f double AP = 0, cur_AP = 0; float IoU_thresh = i_IoU_thresh; + for(int i=0; i &images,const int classes,const f return AP; } -void computeTPFPFN(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh, bool verbose, const bool write_on_file, std::string net) -{ +void computeTPFPFN( std::vector &images,const int classes, + const float IoU_thresh, const float conf_thresh, + bool verbose, const bool write_on_file, std::string net) { std::ofstream out_file; if(write_on_file){ @@ -304,11 +248,10 @@ void computeTPFPFN(std::vector &images,const int classes,const float IoU_ std::vector dets_classes_count(classes,0); std::vector pr(classes); + //compute TP, FP, FN for each image, for each class for(auto &img:images){ - for(auto& tc: truth_classes_count) - tc = 0; - for(auto& dc: dets_classes_count) - dc = 0; + for(auto& tc: truth_classes_count) tc = 0; + for(auto& dc: dets_classes_count) dc = 0; std::vector det_assigned(img.det.size(), false); for(size_t j=0; j &images,const int classes,const float IoU_ int det_index = -1; for(size_t i=0; i conf_thresh){ - float currentIoU = boxIoU(img.det[i], img.gt[j]); + float currentIoU = img.det[i].IoU(img.gt[j]); if(currentIoU > maxIoU && img.det[i].cl == img.gt[j].cl && !det_assigned[i]){ maxIoU = currentIoU; det_index = i; @@ -344,16 +287,14 @@ void computeTPFPFN(std::vector &images,const int classes,const float IoU_ } } + //count all TP, FP, FN and compute precsion, recall and f1-score double avg_precision = 0, avg_recall = 0, f1_score = 0; - - int TP = 0, FP = 0, FN = 0; for(size_t i=0; i 0 ? (double)pr[i].tp / (double)(pr[i].tp +pr[i].fp) : 0; pr[i].recall = (pr[i].tp + pr[i].fn) > 0 ? (double)pr[i].tp / (double)(pr[i].tp +pr[i].fn) : 0; if(verbose) std::cout<<"Class "<