diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 8e4ec3f..f7e9e3e 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -14,352 +14,50 @@ #include "Yolo3Detection.h" #include "CenternetDetection.h" +#include "evaluation.h" + #include -struct BoundigBox : public tk::dnn::box -{ - friend std::ostream& operator<<(std::ostream& os, const BoundigBox& bb); - int unique_truth_index = -1; - int truth_flag = 0; - float max_IoU = 0; - - void clear() - { - unique_truth_index = -1; - truth_flag = 0; - max_IoU = 0; - } -}; - -bool boxComparison (const BoundigBox& a,const BoundigBox& b) -{ - return (a.prob>b.prob); -} - - -std::ostream& operator<<(std::ostream& os, const BoundigBox& bb) -{ - os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << - ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< - bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<< - ", maxIoU: "<< bb.max_IoU<<"\n"; - return os; -} - -struct Frame -{ - void print() const - { - std::cout<<"labels filename: "< gt; - std::vector det; -}; - void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext) { filename.replace(filename.find(l_folder),l_folder.length(),i_folder); filename.replace(filename.find(l_ext),l_ext.length(),i_ext); } -float overlap(float x1, float w1, float x2, float w2) -{ - float l1 = x1 - w1/2; - float l2 = x2 - w2/2; - float left = l1 > l2 ? l1 : l2; - float r1 = x1 + w1/2; - float r2 = x2 + w2/2; - float right = r1 < r2 ? r1 : r2; - return right - left; -} - -float boxIntersection(const BoundigBox &a, const BoundigBox &b) -{ - float w = overlap(a.x, a.w, b.x, b.w); - float h = overlap(a.y, a.h, b.y, b.h); - if(w < 0 || h < 0) - return 0; - float area = w*h; - return area; -} - -float boxUnion(const BoundigBox &a, const BoundigBox &b) -{ - float i = boxIntersection(a, b); - float u = a.w*a.h + b.w*b.h - i; - return u; -} - -float boxIoU(const BoundigBox &a, const BoundigBox &b) -{ - float I = boxIntersection(a, b); - // std::cout<<"I: "< &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false) -{ - std::cout<<"Computing mAP"< truth_classes_count(classes,0); - std::vector dets_classes_count(classes,0); - // std::vector avg_iou_per_class(classes,0); - // std::vector tp_for_thresh_per_class(classes,0); - // std::vector fp_for_thresh_per_class(classes,0); - - - - //count groundtruth and detections in total and for each class - for(auto i:images) - { - for(auto gt:i.gt) - truth_classes_count[gt.cl]++; - for(auto det:i.det) - dets_classes_count[det.cl]++; - detections_count += i.det.size(); - groundtruths_count += i.gt.size(); - } - - std::cout<<"gt_count: "< all_dets; - std::vector all_gts; - - int gt_checked = 0; - - // for each detection comput IoU with groundtruth and match detetcion and - // groundtruth with IoU greater than IoU_thresh - for(auto &img:images) - { - for(size_t i=0; i 0) - { - float maxIoU = 0; - int truth_index = -1; - for(size_t j=0; j maxIoU && img.det[i].cl == img.gt[j].cl) - { - maxIoU = currentIoU; - truth_index = j; - } - } - // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) - { - // std::cout<<"(INSIDE) IoU thresh:"< IoU_thresh:"<<(maxIoU > IoU_thresh)< truth_flags(groundtruths_count,0); - - if(verbose) - for(auto d:all_dets) - std::cout<> pr( classes, std::vector(detections_count)); - for(int rank = 0; rank< detections_count; ++rank) - { - if (rank > 0) - { - for (int class_id = 0; class_id < classes; ++class_id) - { - pr[class_id][rank].tp = pr[class_id][rank - 1].tp; - pr[class_id][rank].fp = pr[class_id][rank - 1].fp; - } - } - - //if it was detected and never detected before - if (all_dets[rank].truth_flag == 1 && truth_flags[all_dets[rank].unique_truth_index] == 0) - { - truth_flags[all_dets[rank].unique_truth_index] = 1; - pr[all_dets[rank].cl][rank].tp++; // true-positive - } - else - { - pr[all_dets[rank].cl][rank].fp++; // false-positive - } - - for (int i = 0; i < classes; ++i) - { - const int tp = pr[i][rank].tp; - const int fp = pr[i][rank].fp; - const int fn = truth_classes_count[i] - tp; // false-negative = objects - true-positive - pr[i][rank].fn = fn; - - if ((tp + fp) > 0) - pr[i][rank].precision = (double)tp / (double)(tp + fp); - else - pr[i][rank].precision = 0; - - if ((tp + fn) > 0) - pr[i][rank].recall = (double)tp / (double)(tp + fn); - else - pr[i][rank].recall = 0; - - if (rank == (detections_count - 1) && dets_classes_count[i] != (tp + fp)) - { // check for last rank - printf(" class_id: %d - detections = %d, tp+fp = %d, tp = %d, fp = %d \n", i, dets_classes_count[i], tp+fp, tp, fp); - } - } - } - - if(verbose) - { - for(int i=0; i < pr.size(); i++) - { - std::cout<<"---------Class "<= 0; --rank) - { - delta_recall = last_recall - pr[i][rank].recall; - last_recall = pr[i][rank].recall; - - if (pr[i][rank].precision > last_precision) - last_precision = pr[i][rank].precision; - - avg_precision += delta_recall * last_precision; - } - } - else //MSCOCO - 101 Recall-points, PascalVOC - 11 Recall-points - { - for (int point = 0; point < map_points; ++point) { - cur_recall = point * 1.0 / ( map_points - 1 ); - cur_precision = 0; - for (int rank = 0; rank < detections_count; ++rank) - if (pr[i][rank].recall >= cur_recall && pr[i][rank].precision > cur_precision) - cur_precision = pr[i][rank].precision; - - avg_precision += cur_precision; - } - avg_precision = avg_precision / map_points; - } - - std::cout<<"Class: "< 1) net = argv[1]; - char type = 'y'; + char ntype = 'y'; if(argc > 2) - type = argv[2][0]; + ntype = argv[2][0]; + //path to txt file with all realpath of images labels char *labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; if(argc > 3) labels_path = argv[3]; - - networkType_t ntype; - switch(type) - { - case 'y': - ntype = YOLO; - break; - case 'c': - ntype = CENTERNET; - break; - default: - FatalError("type not allowed (3rd parameter)"); - } - bool show = false; + bool write_dets = false; tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; + switch(ntype) { - case YOLO: + case 'y': yolo.init(net); break; - case CENTERNET: + case 'c': cnet.init(net); break; default: - FatalError("Network type not allowed "); + FatalError("Network type not allowed (3rd parameter)\n"); } + std::ifstream all_labels(labels_path); std::string l_filename; std::vector images; @@ -372,14 +70,14 @@ int main(int argc, char *argv[]) std::vector detected_bbox; int i=0; - while (std::getline(all_labels, l_filename)) // && i < 1000) + while (std::getline(all_labels, l_filename) && i < 1000) { + std::cout </ / / / - BoundigBox b; + BoundingBox b; b.x = (d.x + d.w/2) / width; b.y = (d.y + d.h/2) / height; b.w = d.w / width; @@ -426,20 +123,22 @@ int main(int argc, char *argv[]) b.cl = d.cl; f.det.push_back(b); - // myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; + if(write_dets) + myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; if(show)// draw rectangle for detection cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); } - // myfile.close(); + if(write_dets) + myfile.close(); // read and save groundtruth labels std::ifstream labels(l_filename); for(std::string line; std::getline(labels, line); ) { std::istringstream in(line); - BoundigBox b; + BoundingBox b; in >> b.cl >> b.x >> b.y >> b.w >> b.h; b.prob = 1; b.truth_flag = 1; @@ -461,23 +160,14 @@ int main(int argc, char *argv[]) std::cout<<"Done."< gt; + std::vector det; + + void print() const; +}; + +struct PR +{ + double precision = 0; + double recall = 0; + int tp = 0, fp = 0, fn = 0; + + void print(); +}; + +float overlap(float x1, float w1, float x2, float w2); +float boxIntersection(const BoundingBox &a, const BoundingBox &b); +float boxUnion(const BoundingBox &a, const BoundingBox &b); +float boxIoU(const BoundingBox &a, const BoundingBox &b); + +double computeMap(std::vector &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false); +double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); + +#endif /*EVALUATION_H*/ \ No newline at end of file diff --git a/src/evaluation.cpp b/src/evaluation.cpp new file mode 100644 index 0000000..2b60449 --- /dev/null +++ b/src/evaluation.cpp @@ -0,0 +1,284 @@ +#include "evaluation.h" + + +void BoundingBox::clear() +{ + unique_truth_index = -1; + truth_flag = 0; + max_IoU = 0; +} + +bool boxComparison (const BoundingBox& a,const BoundingBox& b) +{ + return (a.prob>b.prob); +} + +std::ostream& operator<<(std::ostream& os, const BoundingBox& bb) +{ + os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << + ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< + bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<< + ", maxIoU: "<< bb.max_IoU<<"\n"; + return os; +} + +void Frame::print() const +{ + std::cout<<"labels filename: "< l2 ? l1 : l2; + float r1 = x1 + w1/2; + float r2 = x2 + w2/2; + float right = r1 < r2 ? r1 : r2; + return right - left; +} + +float boxIntersection(const BoundingBox &a, const BoundingBox &b) +{ + float w = overlap(a.x, a.w, b.x, b.w); + float h = overlap(a.y, a.h, b.y, b.h); + if(w < 0 || h < 0) + return 0; + float area = w*h; + return area; +} + +float boxUnion(const BoundingBox &a, const BoundingBox &b) +{ + float i = boxIntersection(a, b); + float u = a.w*a.h + b.w*b.h - i; + return u; +} + +float boxIoU(const BoundingBox &a, const BoundingBox &b) +{ + float I = boxIntersection(a, b); + // std::cout<<"I: "< &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose) +{ + + std::cout<<"Computing mAP"< truth_classes_count(classes,0); + std::vector dets_classes_count(classes,0); + + //count groundtruth and detections in total and for each class + for(auto i:images) + { + for(auto gt:i.gt) + truth_classes_count[gt.cl]++; + for(auto det:i.det) + dets_classes_count[det.cl]++; + detections_count += i.det.size(); + groundtruths_count += i.gt.size(); + } + + std::cout<<"gt_count: "< all_dets; + std::vector all_gts; + + int gt_checked = 0; + + // for each detection comput IoU with groundtruth and match detetcion and + // groundtruth with IoU greater than IoU_thresh + for(auto &img:images) + { + for(size_t i=0; i 0) + { + float maxIoU = 0; + int truth_index = -1; + for(size_t j=0; j maxIoU && img.det[i].cl == img.gt[j].cl) + { + maxIoU = currentIoU; + truth_index = j; + } + } + // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) + { + // std::cout<<"(INSIDE) IoU thresh:"< IoU_thresh:"<<(maxIoU > IoU_thresh)< truth_flags(groundtruths_count,0); + + if(verbose) + for(auto d:all_dets) + std::cout<> pr( classes, std::vector(detections_count)); + for(int rank = 0; rank< detections_count; ++rank) + { + if (rank > 0) + { + for (int class_id = 0; class_id < classes; ++class_id) + { + pr[class_id][rank].tp = pr[class_id][rank - 1].tp; + pr[class_id][rank].fp = pr[class_id][rank - 1].fp; + } + } + + //if it was detected and never detected before + if (all_dets[rank].truth_flag == 1 && truth_flags[all_dets[rank].unique_truth_index] == 0) + { + truth_flags[all_dets[rank].unique_truth_index] = 1; + pr[all_dets[rank].cl][rank].tp++; // true-positive + } + else + { + pr[all_dets[rank].cl][rank].fp++; // false-positive + } + + for (int i = 0; i < classes; ++i) + { + const int tp = pr[i][rank].tp; + const int fp = pr[i][rank].fp; + const int fn = truth_classes_count[i] - tp; // false-negative = objects - true-positive + pr[i][rank].fn = fn; + + if ((tp + fp) > 0) + pr[i][rank].precision = (double)tp / (double)(tp + fp); + else + pr[i][rank].precision = 0; + + if ((tp + fn) > 0) + pr[i][rank].recall = (double)tp / (double)(tp + fn); + else + pr[i][rank].recall = 0; + + if (rank == (detections_count - 1) && dets_classes_count[i] != (tp + fp)) + { // check for last rank + printf(" class_id: %d - detections = %d, tp+fp = %d, tp = %d, fp = %d \n", i, dets_classes_count[i], tp+fp, tp, fp); + } + } + } + + if(verbose) + { + for(int i=0; i < pr.size(); i++) + { + std::cout<<"---------Class "<= 0; --rank) + { + delta_recall = last_recall - pr[i][rank].recall; + last_recall = pr[i][rank].recall; + + if (pr[i][rank].precision > last_precision) + last_precision = pr[i][rank].precision; + + avg_precision += delta_recall * last_precision; + } + } + else //MSCOCO - 101 Recall-points, PascalVOC - 11 Recall-points + { + for (int point = 0; point < map_points; ++point) { + cur_recall = point * 1.0 / ( map_points - 1 ); + cur_precision = 0; + for (int rank = 0; rank < detections_count; ++rank) + if (pr[i][rank].recall >= cur_recall && pr[i][rank].precision > cur_precision) + cur_precision = pr[i][rank].precision; + + avg_precision += cur_precision; + } + avg_precision = avg_precision / map_points; + } + + std::cout<<"Class: "< &images,const int classes,const float i_IoU_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose) +{ + double AP = 0; + float IoU_thresh = i_IoU_thresh; + for(int i=0; i