mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

Pre-process, Process and Post-process work

Browse Source 
Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>
Signed-off-by: Davide Sapienza <sapienza.dav@gmail.com>
This commit is contained in:
Davide Sapienza
2020-01-20 12:27:49 +01:00
parent 23a1365dc4
commit 7838cb4922
10 changed files with 1598 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+1 -1
View File
@@ -27,7 +27,7 @@ if(NVINFER STREQUAL "NVINFER-NOTFOUND")
endif()
# compile
file(GLOB tkdnn_CUSRC "src/kernels/*.cu")
file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/*.cu")
cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${NVINFER_INCLUDES})
cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
demo/demo/demo.cpp
+28 -24
View File
@@ -9,7 +9,8 @@
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "Yolo3Detection.h"
// #include "Yolo3Detection.h"
#include "CenternetDetection.h"
bool gRun;
bool SAVE_RESULT = false;
@@ -25,14 +26,15 @@ int main(int argc, char *argv[]) {
signal(SIGINT, sig_handler);
char *net = "yolo3.rt";
char *net = "resnet101_cnet.rt";
if(argc > 1)
net = argv[1];
char *input = "../demo/yolo_test.mp4";
if(argc > 2)
input = argv[2];
tk::dnn::Yolo3Detection yolo;
// tk::dnn::Yolo3Detection yolo;
tk::dnn::CenternetDetection yolo;
yolo.init(net);
gRun = true;
@@ -66,29 +68,31 @@ int main(int argc, char *argv[]) {
// TODO: async infer
yolo.update(dnn_input);
// draw dets
for(int i=0; i<yolo.detected.size(); i++) {
tk::dnn::box b = yolo.detected[i];
int x0 = b.x;
int x1 = b.x + b.w;
int y0 = b.y;
int y1 = b.y + b.h;
std::string det_class = yolo.getYoloLayer()->classesNames[b.cl];
float prob = b.prob;
frame = yolo.draw(dnn_input);
// // draw dets
// for(int i=0; i<yolo.detected.size(); i++) {
// tk::dnn::box b = yolo.detected[i];
// int x0 = b.x;
// int x1 = b.x + b.w;
// int y0 = b.y;
// int y1 = b.y + b.h;
// std::string det_class = yolo.coco_class_name[b.cl];
// // yolo.getYoloLayer()->classesNames[b.cl];
// float prob = b.prob;
std::cout<<det_class<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
// draw rectangle
cv::rectangle(frame, cv::Point(x0, y0), cv::Point(x1, y1), yolo.colors[b.cl], 2);
// // std::cout<<det_class<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
// // draw rectangle
// cv::rectangle(frame, cv::Point(x0, y0), cv::Point(x1, y1), yolo.colors[b.cl], 2);
// draw label
int baseline = 0;
float fontScale = 0.5;
int thickness = 2;
cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline);
cv::rectangle(frame, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), yolo.colors[b.cl], -1);
cv::putText(frame, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness);
}
// // draw label
// int baseline = 0;
// float fontScale = 0.5;
// int thickness = 2;
// cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline);
// cv::rectangle(frame, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), yolo.colors[b.cl], -1);
// cv::putText(frame, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness);
// }
cv::imshow("detection", frame);
cv::waitKey(1);
if(SAVE_RESULT)
include/sorting.h
+19
View File
@@ -0,0 +1,19 @@
#include <thrust/sort.h>
#include <thrust/execution_policy.h>
#include <thrust/functional.h>
#include <thrust/transform.h>
#include <thrust/iterator/constant_iterator.h>
#include <thrust/gather.h>
#include <thrust/copy.h>
#include "tkdnn.h"
void sort(dnnType *src_begin, dnnType *src_end, int *idsrc);
void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores,
int *topk_inds, float *topk_ys, float *topk_xs);
void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes);
void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out);
void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys);
void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, float *xs_begin, float *ys_begin, dnnType *src_begin);
void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1);
include/tkDNN/CenternetDetection.h
+112
View File
@@ -0,0 +1,112 @@
#include <iostream>
#include <cstring>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#include <unistd.h>
#include <mutex>
#include "utils.h"
#include <time.h>
#include "kernels.h"
#include <vector>
#include <numeric> // std::iota
#include <algorithm> // std::sort
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "tkdnn.h"
#include "sorting.h"
namespace tk { namespace dnn {
/**
*
* @author Francesco Gatti
*/
class CenternetDetection {
private:
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_h, *input, *input_d;
int ndets = 0;
// tk::dnn::Yolo::detection *dets = nullptr;
cv::Mat imageF;
cv::Mat bgr[3];
// variable to test cnet on dog pictures
tk::dnn::dataDim_t dim;
tk::dnn::dataDim_t dim2;
cv::Size sz;
const char *input_bin = "../tests/resnet101_cnet/debug/input.bin";
// pre-process
tk::dnn::dataDim_t dim_hm;
tk::dnn::dataDim_t dim_wh;
tk::dnn::dataDim_t dim_reg;
float *topk_scores;
int *topk_inds_;
float *topk_ys_;
float *topk_xs_;
int *ids_d, *ids_, *ids_2, *ids_2d;
float *scores, *scores_d;
int *clses, *clses_d;
int *topk_inds_d;
float *topk_ys_d;
float *topk_xs_d;
int *inttopk_xs_d, *inttopk_ys_d;
float *bbx0, *bby0, *bbx1, *bby1;
float *bbx0_d, *bby0_d, *bbx1_d, *bby1_d;
float *target_coords;
cv::Vec<float, 3> mean;
cv::Vec<float, 3> stddev;
cv::Mat src;
cv::Mat dst;
//processing
float toll = 0.000001;
int K = 100;
int width = 56; // TODO
public:
dnnType *rt_out[4];
float inp_height = 224;//512;
float inp_width = 224;//512;
int classes = 80;
int num = 0;
int n_masks = 0;
float thresh = 0.3;
cv::Scalar colors[256];
// this is filled with results
std::vector<tk::dnn::box> detected;
// draw
std::vector<std::string> coco_class_name;
CenternetDetection() {}
virtual ~CenternetDetection() {}
/**
* Method used for inizialize the class
*
* @return Success of the initialization
*/
bool init(std::string tensor_path);
void testdog();
cv::Mat draw(cv::Mat &frame);
void update(cv::Mat &frame);
};
}}
include/tkDNN/Layer.h
+1 -1
View File
@@ -292,7 +292,7 @@ public:
Pooling(Network *net, int winH, int winW,
int strideH, int strideW,
int paddingH = 0, int paddingW = 0,
tkdnnPoolingMode_t pool_mode = POOLING_MAX);
tkdnnPoolingMode_t pool_mode = POOLING_MAX, bool final = false);
virtual ~Pooling();
virtual layerType_t getLayerType() { return LAYER_POOLING; };
src/CenternetDetection.cpp
+617
View File
@@ -0,0 +1,617 @@
#include "CenternetDetection.h"
namespace tk { namespace dnn {
float __colors[6][3] = { {1,0,1}, {0,0,1},{0,1,1},{0,1,0},{1,1,0},{1,0,0} };
float get_color2(int c, int x, int max)
{
float ratio = ((float)x/max)*5;
int i = floor(ratio);
int j = ceil(ratio);
ratio -= i;
float r = (1-ratio) * __colors[i % 6][c % 3] + ratio*__colors[j % 6][c % 3];
//printf("%f\n", r);
return r;
}
bool CenternetDetection::init(std::string tensor_path) {
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
dim = tk::dnn::dataDim_t(1, 3, 224, 224, 1);
const char *coco_class_name_[] = {
"person", "bicycle", "car", "motorcycle", "airplane",
"bus", "train", "truck", "boat", "traffic light", "fire hydrant",
"stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse",
"sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
"umbrella", "handbag", "tie", "suitcase", "frisbee", "skis",
"snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
"skateboard", "surfboard", "tennis racket", "bottle", "wine glass",
"cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich",
"orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake",
"chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv",
"laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
"oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
"scissors", "teddy bear", "hair drier", "toothbrush"
};
coco_class_name = std::vector<std::string>(coco_class_name_, std::end( coco_class_name_ ));
src = cv::Mat(cv::Size(2,3), CV_32F);
dst = cv::Mat(cv::Size(2,3), CV_32F);
// dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
checkCuda(cudaMallocHost(&input_h, sizeof(dnnType)*netRT->input_dim.tot()));
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot()));
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot()));
dim_hm = tk::dnn::dataDim_t(1, 80, 56, 56, 1);
dim_wh = tk::dnn::dataDim_t(1, 2, 56, 56, 1);
dim_reg = tk::dnn::dataDim_t(1, 2, 56, 56, 1);
checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
checkCuda( cudaMalloc(&ids_2d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
checkCuda( cudaMallocHost(&ids_2, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
for(int i =0; i<dim_hm.c * dim_hm.h * dim_hm.w; i++){
ids_[i] = i;
}
int val = 0;
for(int i =0; i <dim_hm.c * dim_hm.h * dim_hm.w; i++){
ids_2[i] = val;
if(i%dim_hm.c == 0)
val = 0;
}
checkCuda( cudaMallocHost(&scores, K *sizeof(float)) );
checkCuda( cudaMalloc(&scores_d, K *sizeof(float)) );
checkCuda( cudaMallocHost(&clses, K *sizeof(int)) );
checkCuda( cudaMalloc(&clses_d, K *sizeof(int)) );
// checkCuda( cudaMallocHost(&topk_inds, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_inds_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_d, K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_d, K *sizeof(float)) );
// checkCuda( cudaMalloc(&intid, K *sizeof(int)) );
checkCuda( cudaMalloc(&inttopk_ys_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&inttopk_xs_d, K *sizeof(int)) );
// checkCuda( cudaMalloc(&ids_d, dim_hm.c * K*sizeof(int)) );
// checkCuda( cudaMallocHost(&wh_aus, dim_wh.tot()*sizeof(dnnType)) );
checkCuda( cudaMallocHost(&bbx0, K * sizeof(float)) );
checkCuda( cudaMallocHost(&bby0, K * sizeof(float)) );
checkCuda( cudaMallocHost(&bbx1, K * sizeof(float)) );
checkCuda( cudaMallocHost(&bby1, K * sizeof(float)) );
checkCuda( cudaMalloc(&bbx0_d, K * sizeof(float)) );
checkCuda( cudaMalloc(&bby0_d, K * sizeof(float)) );
checkCuda( cudaMalloc(&bbx1_d, K * sizeof(float)) );
checkCuda( cudaMalloc(&bby1_d, K * sizeof(float)) );
checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) );
mean << 0.408, 0.447, 0.47;
stddev << 0.289, 0.274, 0.278;
}
void CenternetDetection::testdog() {
readBinaryFile(input_bin, dim.tot(), &input_h, &input_d);
// -------- transofrm compose
cv::Mat imageORIG = cv::imread("../../dog.jpg");
imageORIG.convertTo(imageF, CV_32FC3, 1/255.0);
sz = imageF.size();
std::cout<<"image: "<<sz.width<<", "<<sz.height<<std::endl;
resize(imageF, imageF, cv::Size(256, 256));
const int cropSize = 224;
const int offsetW = (imageF.cols - cropSize) / 2;
const int offsetH = (imageF.rows - cropSize) / 2;
const cv::Rect roi(offsetW, offsetH, cropSize, cropSize);
imageF = imageF(roi).clone();
std::cout << "Cropped image dimension: " << imageF.cols << " X " << imageF.rows << std::endl;
mean << 0.485, 0.456, 0.406;
stddev << 0.229, 0.224, 0.225;
sz = imageF.size();
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
// std::cout<<"mean: "<<mean<<", std: "<<stddev<<std::endl;
cv::add(imageF, -mean, imageF);
cv::divide(imageF, stddev, imageF);
//split channels
cv::split(imageF,bgr);//split source
dim2 = dim;
//write channels
for(int i=0; i<dim2.c; i++) {
int idx = i*imageF.rows*imageF.cols;
int ch = dim2.c-1 -i;
memcpy((void*)&input[idx], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
printCenteredTitle(" TENSORRT inference ", '=', 30); {
dim2.print();
TIMER_START
netRT->infer(dim2, input_d);
TIMER_STOP
dim2.print();
}
// checkResult(dim2.tot(), input_h, input);
}
cv::Mat CenternetDetection::draw(cv::Mat &imageORIG) {
tk::dnn::box b;
int x0, w, x1, y0, h, y1;
int objClass;
std::string det_class;
int baseline = 0;
float fontScale = 0.5;
int thickness = 2;
for(int c=0; c<classes; c++) {
int offset = c*123457 % classes;
float r = get_color2(2, offset, classes);
float g = get_color2(1, offset, classes);
float b = get_color2(0, offset, classes);
colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
}
int num_detected = detected.size();
for (int i = 0; i < num_detected; i++){
b = detected[i];
x0 = b.x;
w = b.w;
x1 = b.x + w;
y0 = b.y;
h = b.h;
y1 = b.y + h;
objClass = b.cl;
det_class = coco_class_name[objClass];
cv::rectangle(imageORIG, cv::Point(x0, y0), cv::Point(x1, y1), colors[objClass], 2);
// draw label
cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline);
cv::rectangle(imageORIG, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), colors[b.cl], -1);
cv::putText(imageORIG, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness);
}
return imageORIG;
// cv::namedWindow("cnet", cv::WINDOW_NORMAL);
// cv::imshow("cnet", imageOrig);
// cv::waitKey(10000);
}
void CenternetDetection::update(cv::Mat &imageORIG) {
if(!imageORIG.data) {
std::cout<<"YOLO: NO IMAGE DATA\n";
return;
}
TIMER_START
auto start_t = std::chrono::steady_clock::now();
auto step_t = std::chrono::steady_clock::now();
auto end_t = std::chrono::steady_clock::now();
// -----------------------------------pre-process ------------------------------------------
// it will resize the images to `224 x 224` in GETTING_STARTED.md
cv::Size sz = imageORIG.size();
std::cout<<"image: "<<sz.width<<", "<<sz.height<<std::endl;
float scale = 1.0;
float new_height = sz.height * scale;
float new_width = sz.width * scale;
float c[] = {new_width / 2.0, new_height /2.0};
float s[2];
if(sz.width > sz.height){
s[0] = sz.width * 1.0;
s[1] = sz.width * 1.0;
}
else{
s[0] = sz.height * 1.0;
s[1] = sz.height * 1.0;
}
// ----------- get_affine_transform
// rot_rad = pi * 0 / 100 --> 0
src.at<float>(0,0)=c[0];
src.at<float>(0,1)=c[1];
src.at<float>(1,0)=c[0];
src.at<float>(1,1)=c[1] + s[0] * -0.5;
dst.at<float>(0,0)=inp_width * 0.5;
dst.at<float>(0,1)=inp_height * 0.5;
dst.at<float>(1,0)=inp_width * 0.5;
dst.at<float>(1,1)=inp_height * 0.5 + inp_width * -0.5;
src.at<float>(2,0)=src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
src.at<float>(2,1)=src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
dst.at<float>(2,0)=dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
dst.at<float>(2,1)=dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
// std::cout<<"src: "<<src<<std::endl;
// std::cout<<"dst: "<<dst<<std::endl;
cv::Mat trans = cv::getAffineTransform( src, dst );
end_t = std::chrono::steady_clock::now();
std::cout << " TIME getAffinetr : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
resize(imageORIG, imageF, cv::Size(new_width, new_height));
sz = imageF.size();
std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
end_t = std::chrono::steady_clock::now();
std::cout << " TIME resize: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
cv::warpAffine(imageF, imageF, trans, cv::Size(inp_width, inp_height), cv::INTER_LINEAR );
end_t = std::chrono::steady_clock::now();
std::cout << " TIME warpAffine: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
sz = imageF.size();
std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
imageF.convertTo(imageF, CV_32FC3, 1/255.0);
end_t = std::chrono::steady_clock::now();
std::cout << " TIME convert_to: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
std::cout<<"mean: "<<mean<<", std: "<<stddev<<std::endl;
dim2 = dim;
end_t = std::chrono::steady_clock::now();
std::cout << " TIME before split: " << std::chrono::duration_cast<std::chrono::microseconds>(end_t - step_t).count() << " us" << std::endl;
step_t = end_t;
//split channels
cv::split(imageF,bgr);//split source
end_t = std::chrono::steady_clock::now();
std::cout << " TIME split: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
for(int i=0; i<3; i++){
bgr[i] = bgr[i] - mean[i];
bgr[i] = bgr[i] / stddev[i];
}
end_t = std::chrono::steady_clock::now();
std::cout << " TIME mean std: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
//write channels
for(int i=0; i<dim2.c; i++) {
int idx = i*imageF.rows*imageF.cols;
int ch = dim2.c-3 +i;
std::cout<<"i: "<<i<<", idx: "<<idx<<", ch: "<<ch<<std::endl;
memcpy((void*)&input[idx], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
printCenteredTitle(" TENSORRT inference ", '=', 30); {
dim2.print();
TIMER_START
netRT->infer(dim2, input_d);
TIMER_STOP
dim2.print();
}
// checkResult(dim2.tot(), input_h, input);
std::cout<<" --- pre-process ---\n";
end_t = std::chrono::steady_clock::now();
std::cout << " TIME : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// ------------------------------------ process --------------------------------------------
rt_out[0] = (dnnType *)netRT->buffersRT[1];
rt_out[1] = (dnnType *)netRT->buffersRT[2];
rt_out[2] = (dnnType *)netRT->buffersRT[3];
rt_out[3] = (dnnType *)netRT->buffersRT[4];
activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot());
checkCuda( cudaDeviceSynchronize() );
end_t = std::chrono::steady_clock::now();
std::cout << " TIME sigmoid : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0]);
float *prova;
checkCuda( cudaMallocHost(&prova, K*sizeof(float)) );
checkCuda( cudaMemcpy(prova, rt_out[0], K*sizeof(float), cudaMemcpyDeviceToHost) );
std::cout<<"heat:\n";
for(int i=0; i<K; i++)
std::cout<<prova[i]<<" ";
std::cout<<"\n\n\n";
// for(int i=0; i < dim_hm.tot(); i++){
// if(hm_h[i]-hmax_h[i] > toll || hm_h[i]-hmax_h[i] < -toll){
// hm_h[i] = 0.0f;
// }
// }
// checkCuda( cudaFreeHost(hmax_h) );
std::cout<<" --- hmax ---\n";
end_t = std::chrono::steady_clock::now();
std::cout << " TIME : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// ----------- nms end
// ----------- topk
// thrust::device_vector<int> ids_d;
// int ids[dim_hm.h * dim_hm.w];
// for(int i=0; i<dim_hm.h * dim_hm.w; i++){
// ids[i]=i;
// }
// std::vector<int> ids2( dim_hm.h * dim_hm.w );
// for(int i=0; i<dim_hm.h * dim_hm.w; i++){
// ids2[i]=i;
// }
// int ids2[dim_hm.h * dim_hm.w];
// checkCuda( cudaMemcpy(ids2_d, ids2, dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );
if(K > dim_hm.h * dim_hm.w){
printf ("Error topk (K is too large)\n");
return;
}
checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );
// checkCuda( cudaMemcpy(ids_2d, ids_2, dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );
// sortAndTopKonDevice(rt_out[0], ids_2d, topk_scores, topk_inds_ , topk_ys_ , topk_xs_ ,dim_hm.h * dim_hm.w, K, dim_hm.c);
// checkCuda( cudaDeviceSynchronize() );
// for(int i=0; i<dim_hm.c; i++){
// // get the hm->output_dim.h * hm->output_dim.w elements for each channel and sort it. Then find the first 100 elements
// // memcpy(ids2, ids, dim_hm.h * dim_hm.w);
// sort(rt_out[0]+ i * dim_hm.h * dim_hm.w,
// rt_out[0]+ i * dim_hm.h * dim_hm.w + dim_hm.h * dim_hm.w,
// ids_d);
// // end_t = std::chrono::steady_clock::now();
// // std::cout << " TIME sort channel "<<i<<": " << std::chrono::duration_cast<std::chrono::microseconds>(end_t - step_t).count() << " ms" << std::endl;
// // step_t = end_t;
// topk(rt_out[0]+ i * dim_hm.h * dim_hm.w, ids_d, K, topk_scores + i*K,
// topk_inds_ + i*K, topk_ys_ + i*K, topk_xs_ + i*K);
// // checkCuda( cudaMemcpy(ids2, ids2_d, dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyDeviceToHost) );
// // for (int j=0; j<dim_hm.h * dim_hm.w; j++) {
// // topk_scores[i*K + count] = hm_h[i * dim_hm.h * dim_hm.w + ids2[j]];
// // topk_inds_[i*K +count] = ids2[j];
// // topk_ys_[i*K +count] = (int)(ids2[j] / width);
// // topk_xs_[i*K +count] = (int)(ids2[j] % width);
// // if(++count == K)
// // break;
// // }
// // end_t = std::chrono::steady_clock::now();
// // std::cout << " TIME topk channel "<<i<<": " << std::chrono::duration_cast<std::chrono::microseconds>(end_t - step_t).count() << " ms" << std::endl;
// // step_t = end_t;
// }
// checkCuda( cudaFree(ids_d ));
std::cout<<" --- a 100 ---\n";
end_t = std::chrono::steady_clock::now();
std::cout << " TIME sort topk on 80 channel: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// final
// sort(topk_scores,
// topk_scores + dim_hm.c * K,
// topk_inds_);
sort(rt_out[0],
rt_out[0]+dim_hm.tot(),
ids_d);
checkCuda( cudaDeviceSynchronize() );
int *topk_inds;
checkCuda( cudaMallocHost(&topk_inds, K*sizeof(int)) );
// checkCuda( cudaMemcpy(topk_inds, ids_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
// for(int i=0; i<K; i++)
// std::cout<<topk_inds[i]<<" ";
// std::cout<<"\n\n\n";
end_t = std::chrono::steady_clock::now();
std::cout << " TIME sort channel: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// topk(topk_scores, topk_inds_, K, scores_d,
// topk_inds_d, topk_ys_d, topk_xs_d);
topk(rt_out[0], ids_d, K, scores_d,
topk_inds_d, topk_ys_d, topk_xs_d);
checkCuda( cudaDeviceSynchronize() );
end_t = std::chrono::steady_clock::now();
std::cout << " TIME topk channel: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
checkCuda( cudaMemcpy(topk_inds, topk_inds_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
for(int i=0; i<K; i++)
std::cout<<topk_inds[i]<<" ";
std::cout<<std::endl;
checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) );
std::cout<<"\n\nscores:\n";
for(int i=0; i<K;i++)
std::cout<<scores[i]<<" ";
std::cout<<std::endl;
std::cout<<"\n\n\n";
topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d);
checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
std::cout<<"\ntopk_ids: \n";
checkCuda( cudaMemcpy(topk_inds, topk_inds_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
for(int i=0; i<K; i++)
std::cout<<topk_inds[i]<<" ";
std::cout<<std::endl;
std::cout<<"\ntopk_clses: \n";
checkCuda( cudaMemcpy(topk_inds, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
for(int i=0; i<K; i++)
std::cout<<topk_inds[i]<<" ";
std::cout<<std::endl;
std::cout<<"\nxs: \n";
checkCuda( cudaMemcpy(topk_inds, topk_xs_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
for(int i=0; i<K; i++)
std::cout<<topk_inds[i]<<" ";
std::cout<<std::endl;
std::cout<<"\nys: \n";
checkCuda( cudaMemcpy(topk_inds, topk_ys_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
for(int i=0; i<K; i++)
std::cout<<topk_inds[i]<<" ";
std::cout<<std::endl;
// return;
// checkCuda( cudaDeviceSynchronize() );
// checkCuda( cudaFree(topk_scores) );
// checkCuda( cudaFree(topk_inds_) );
// checkCuda( cudaFree(topk_ys_) );
// checkCuda( cudaFree(topk_xs_) );
// checkCuda( cudaFree(scores_d) );
// checkCuda( cudaFree(topk_inds_d) );
end_t = std::chrono::steady_clock::now();
std::cout << " TIME clses topk 1 time: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// ----------- topk end
// dnnType *reg_aus;
// checkCuda( cudaMallocHost(&reg_aus, dim_reg.tot()*sizeof(dnnType)) );
// checkCuda( cudaMemcpy(reg_aus, rt_out[3], dim_reg.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) );
// for(int i = 0; i < K; i++){
// topk_xs[i] = topk_xs[i] + reg_aus[topk_inds[i]];
// topk_ys[i] = topk_ys[i] + reg_aus[topk_inds[i]+dim_reg.h*dim_reg.w];
// }
topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3]);
// checkCuda( cudaDeviceSynchronize() );
end_t = std::chrono::steady_clock::now();
std::cout << " TIME add offset: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// checkCuda( cudaFreeHost(reg_aus) );
// dnnType *wh_aus;
// checkCuda( cudaMemcpy(wh_aus, rt_out[2], dim_wh.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) );
bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d);
// checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(bby0, bby0_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(bbx1, bbx1_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
// for(int i = 0; i < K; i++){
// bboxes[i * 4] = topk_xs[i] - wh_aus[topk_inds[i]] / 2;
// bboxes[i * 4 + 1] = topk_ys[i] - wh_aus[topk_inds[i]+dim_reg.h*dim_reg.w] / 2;
// bboxes[i * 4 + 2] = topk_xs[i] + wh_aus[topk_inds[i]] / 2;
// bboxes[i * 4 + 3] = topk_ys[i] + wh_aus[topk_inds[i]+dim_reg.h*dim_reg.w] / 2;
// }
// for(int i = 0; i < K; i++){
// std::cout<<"-----\n(x0, y0) = ("<<bbx0<<", "<<bby0<<")\n(x1,y1) = ("<<bbx1<<", "<<bby1<<")\n";
// }
// checkCuda( cudaFreeHost(wh_aus) );
// checkCuda( cudaFreeHost(topk_inds) );
// checkCuda( cudaFreeHost(topk_ys) );
// checkCuda( cudaFreeHost(topk_xs) );
std::cout<<" --- bboxes ---\n";
end_t = std::chrono::steady_clock::now();
std::cout << " TIME : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// servono [bboxes, scores, clses]
// checkCuda( cudaDeviceSynchronize() );
std::cout<<" --- process ---\n";
end_t = std::chrono::steady_clock::now();
std::cout << " TIME : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
// ---------------------------------- post-process -----------------------------------------
// --------- ctdet_post_process
// --------- transform_preds
src.at<float>(0,0)=c[0];
src.at<float>(0,1)=c[1];
src.at<float>(1,0)=c[0];
src.at<float>(1,1)=c[1] + s[0] * -0.5;
dst.at<float>(0,0)=width * 0.5;
dst.at<float>(0,1)=width * 0.5;
dst.at<float>(1,0)=width * 0.5;
dst.at<float>(1,1)=width * 0.5 + width * -0.5;
src.at<float>(2,0)=src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
src.at<float>(2,1)=src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
dst.at<float>(2,0)=dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
dst.at<float>(2,1)=dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
cv::Mat trans2(cv::Size(3,2), CV_32F);
trans2 = cv::getAffineTransform( dst, src );
end_t = std::chrono::steady_clock::now();
std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
cv::Mat new_pt1(cv::Size(1,2), CV_32F);
cv::Mat new_pt2(cv::Size(1,2), CV_32F);
for(int i = 0; i<K; i++){
new_pt1.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*bbx0[i] +
static_cast<float>(trans2.at<double>(0,1))*bby0[i] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt1.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*bbx0[i] +
static_cast<float>(trans2.at<double>(1,1))*bby0[i] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
new_pt2.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*bbx1[i] +
static_cast<float>(trans2.at<double>(0,1))*bby1[i] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt2.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*bbx1[i] +
static_cast<float>(trans2.at<double>(1,1))*bby1[i] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
// std::cout<<"\n new: "<<new_pt1<<" - "<<new_pt2<<std::endl;
target_coords[i*4] = new_pt1.at<float>(0,0);
target_coords[i*4+1] = new_pt1.at<float>(0,1);
target_coords[i*4+2] = new_pt2.at<float>(0,0);
target_coords[i*4+3] = new_pt2.at<float>(0,1);
// std::cout<<new_pt1.at<float>(0,0)<<", "<<new_pt1.at<float>(0,1)<<", "<<new_pt2.at<float>(0,0)<<", "<<new_pt2.at<float>(0,1)<<std::endl;
// std::cout<<"target:cords "<<target_coords[i*4]<<" - "<<target_coords[i*4+1]<<std::endl;
}
// int *classes;
detected.clear();
for(int i = 0; i<classes; i++){
for(int j=0; j<K; j++)
if(clses[j] == i){
if(scores[j] > thresh){
std::cout<<"th: "<<scores[j]<<" - cl: "<<clses[j]<<" i: "<<i<<std::endl;
//add coco bbox
//det[0:4], i, det[4]
int x0 = target_coords[j*4];
int y0 = target_coords[j*4+1];
int x1 = target_coords[j*4+2];
int y1 = target_coords[j*4+3];
int obj_class = clses[j];
float prob = scores[j];
// std::cout<<"("<<x0<<", "<<y0<<"),("<<x1<<", "<<y1<<")"<<std::endl;
tk::dnn::box res;
res.cl = obj_class;
res.prob = prob;
res.x = x0;
res.y = y0;
res.w = x1 - x0;
res.h = y1 - y0;
detected.push_back(res);
}
}
}
std::cout<<" --- post_process ---\n";
end_t = std::chrono::steady_clock::now();
std::cout << " TIME : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
step_t = end_t;
std::cout<<"TOTAL: \n";
TIMER_STOP
}
}}
src/NetworkRT.cpp
+6 -1
View File
@@ -292,7 +292,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) {
if(l->pool_mode == tkdnnPoolingMode_t::POOLING_AVERAGE_EXCLUDE_PADDING) ptype = PoolingType::kMAX_AVERAGE_BLEND;
if(l->input_dim.h % 2 == 1 && l->input_dim.w % 2 == 1)
// if(l->input_dim.h % 2 == 1 && l->input_dim.w % 2 == 1)
if(l->input_dim.h == l->output_dim.h && l->input_dim.w == l->output_dim.w)
{
IPlugin *plugin = new ResizeLayerRT( l->output_dim.c,l->output_dim.h+1,l->output_dim.w+1 );
IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
@@ -324,6 +325,10 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) {
IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kRELU);
checkNULL(lRT);
return lRT;
} else if(l->act_mode == CUDNN_ACTIVATION_SIGMOID) {
IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kSIGMOID);
checkNULL(lRT);
return lRT;
} else {
FatalError("this Activation mode is not yet implemented");
src/Pooling.cpp
+13 -6
View File
@@ -7,8 +7,8 @@ namespace tk { namespace dnn {
Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW,
int paddingH, int paddingW,
tkdnnPoolingMode_t pool_mode) :
Layer(net) {
tkdnnPoolingMode_t pool_mode, bool final) :
Layer(net, final) {
this->winH = winH;
this->winW = winW;
@@ -53,10 +53,17 @@ Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW,
//compute w and h as in darknet
int padH = paddingH == 0? winH -1 : paddingH;
int padW = paddingW == 0? winW -1 : paddingW;
h = (h + padH - winH)/strideH +1;
w = (w + padW - winW)/strideW +1;
if(final){
h = (h + padH - winH)/strideH +1 +1;
w = (w + padW - winW)/strideW +1 +1;
}
else{
h = (h + padH - winH)/strideH +1;
w = (w + padW - winW)/strideW +1;
}
// h = (h + winH*this->paddingH)/strideH;
// w = (w + winW*this->paddingW)/strideW;
src/sorting.cu
+104
View File
@@ -0,0 +1,104 @@
#include "sorting.h"
void sort(dnnType *src_begin, dnnType *src_end, int *idsrc)
{
thrust::sort_by_key(thrust::device,
src_begin, src_end, idsrc,
thrust::greater<float>());
// thrust::stable_sort_by_key(thrust::device,
// src_begin, src_end, idsrc,
// thrust::greater<float>());
}
void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores,
int *topk_inds, float *topk_ys, float *topk_xs)
{
checkCuda( cudaMemcpy(topk_scores, (float *)src_begin, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(topk_inds, idsrc, K*sizeof(int), cudaMemcpyDeviceToDevice) );
// topk_ys_[i*K +count] = (int)(ids2[j] / width);
// topk_xs_[i*K +count] = (int)(ids2[j] % width);
}
__global__
void sortAndTopK_kernel(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs,const int size, const int K){
int i = blockDim.x*blockIdx.x + threadIdx.x;
thrust::sort_by_key(thrust::device, src_begin + i * size, src_begin + i * size + size, idsrc + i * size, thrust::greater<float>());
thrust::copy_n(thrust::device, src_begin + i * size, K, topk_scores + i * K);
// thrust::copy_n(thrust::device, idsrc + i * size, K, topk_inds + i * K );
thrust::copy_n(thrust::device, idsrc + i * size, K, topk_inds + i * K );
}
void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes)
{
int blocks = n_classes;
int threads = 1;
sortAndTopK_kernel<<<blocks, threads, 0>>>(src_begin, idsrc, topk_scores, topk_inds, topk_ys, topk_xs, size, K);
}
struct threshold : public thrust::binary_function<float,float,float>
{
__host__ __device__
float operator()(float x, float y) {
float toll = 1e-6;
if(fabsf(x-y)>toll)
return 0.0f;
else
return x;
}
};
void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out){
struct threshold op;
thrust::transform(thrust::device, src_begin, src_end, src2_begin, src_out, op);
}
void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys){
thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), clses, thrust::divides<int>());
thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), ids_begin, thrust::modulus<int>());
thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(size), ys, thrust::divides<int>());
thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(size), xs, thrust::modulus<int>());
}
void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, float *xs_begin, float *ys_begin, dnnType *src_begin){
float *src_out;
checkCuda( cudaMalloc(&src_out, K *sizeof(float)) );
thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out);
thrust::transform(thrust::device, intxs_begin, intxs_begin + K, src_out, xs_begin, thrust::plus<float>());
int *ids_out;
checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) );
thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(size), ids_out, thrust::plus<int>());
thrust::gather(thrust::device, ids_out, ids_out+K, src_begin, src_out);
thrust::transform(thrust::device, intys_begin, intys_begin + K, src_out, ys_begin, thrust::plus<float>());
checkCuda( cudaFree(src_out) );
checkCuda( cudaFree(ids_out) );
}
void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1){
float *src_out;
checkCuda( cudaMalloc(&src_out, K *sizeof(float)) );
thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out);
thrust::transform(thrust::device, src_out, src_out + K, thrust::make_constant_iterator(2), src_out, thrust::divides<float>());
// x0
thrust::transform(thrust::device, xs_begin, xs_begin + K, src_out, bbx0, thrust::minus<float>());
// x1
thrust::transform(thrust::device, xs_begin, xs_begin + K, src_out, bbx1, thrust::plus<float>());
int *ids_out;
checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) );
thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(size), ids_out, thrust::plus<int>());
thrust::gather(thrust::device, ids_out, ids_out + K, src_begin, src_out);
thrust::transform(thrust::device, src_out, src_out + K, thrust::make_constant_iterator(2), src_out, thrust::divides<float>());
// y0
thrust::transform(thrust::device, ys_begin, ys_begin + K, src_out, bby0, thrust::minus<float>());
// y1
thrust::transform(thrust::device, ys_begin, ys_begin + K, src_out, bby1, thrust::plus<float>());
checkCuda( cudaFree(src_out) );
checkCuda( cudaFree(ids_out) );
}
tests/resnet101_cnet/resnet101_cnet.cpp
+697 -6
View File
@@ -1,5 +1,12 @@
#include <iostream>
#include "kernels.h"
#include "Yolo3Detection.h"
#include "tkdnn.h"
#include <vector>
#include <numeric> // std::iota
#include <algorithm> // std::sort
// #include "utils.h"
const char *input_bin = "../tests/resnet101_cnet/debug/input.bin";
const char *conv1_bin = "../tests/resnet101_cnet/layers/conv1.bin";
@@ -172,9 +179,56 @@ const char *reg_conv2_bin = "../tests/resnet101_cnet/layers/reg-2.bin";
const char *fc_bin = "../tests/resnet101_cnet/layers/fc.bin";
const char *output_bin[]={
"../tests/resnet101_cnet/debug/hm.bin",
"../tests/resnet101_cnet/debug/wh.bin",
"../tests/resnet101_cnet/debug/reg.bin"};
"../tests/resnet101_cnet/debug/hm.bin",
"../tests/resnet101_cnet/debug/wh.bin",
"../tests/resnet101_cnet/debug/reg.bin"};
std::vector<size_t> sort_indexes(const std::vector<float> &v) {
// initialize original index locations
std::vector<size_t> idx(v.size());
iota(idx.begin(), idx.end(), 0);
// sort indexes based on comparing values in v
sort(idx.begin(), idx.end(),
[&v](size_t i1, size_t i2) {return v[i1] > v[i2];});
return idx;
}
float _colors[6][3] = { {1,0,1}, {0,0,1},{0,1,1},{0,1,0},{1,1,0},{1,0,0} };
float get_color(int c, int x, int max)
{
float ratio = ((float)x/max)*5;
int i = floor(ratio);
int j = ceil(ratio);
ratio -= i;
float r = (1-ratio) * _colors[i % 6][c % 3] + ratio*_colors[j % 6][c % 3];
//printf("%f\n", r);
return r;
}
int computeDetections(dnnType *hm_d, dnnType *wh_d, dnnType *reg_d, int hm_dim, int wh_dim, int reg_dim, bool cat_spec_wh, int k){
// _nms
int kernel = 3;
int pad = (kernel - 1)/2;
std::cout<<"computeDetections\n";
// dnnType *hmax;
// tk::dnn::Pooling maxpool(&hmax, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX)
// = (dnnType *)
// net.functional.max_pool2d(
// heat, (kernel, kernel), stride=1, padding=pad)
// keep = (hmax == heat).float()
// return heat * keep
}
int process(dnnType *hm_d, dnnType *wh_d, dnnType *reg_d, int hm_dim, int wh_dim, int reg_dim){
std::cout<<"process\n";
// computeDetections(hm_d, wh_d, reg_d, hm_dim, wh_dim, reg_dim, false, 100);
}
int main()
{
@@ -211,7 +265,6 @@ int main()
last = layer1_0_relu;
}
// tk::dnn::Activation *last_activation = (tk::dnn::Activation *) net.layers[net.num_layers-1];
// layer 2
int id_layer2_bin = 0;
for(int i=0; i<4;i++)
@@ -318,6 +371,10 @@ int main()
tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false);
tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 80, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false, false, true);
int kernel = 3;
int pad = (kernel - 1)/2;
tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID);
tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX, true);
tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false);
@@ -365,7 +422,7 @@ int main()
}
tk::dnn::Layer *outs[3] = { hm, wh, reg };
int out_count = 1;
for(int i=0; i<3; i++) {
printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30);
@@ -381,7 +438,10 @@ int main()
dnnType *cudnn_out, *rt_out;
cudnn_out = outs[i]->dstData;
rt_out = (dnnType *)netRT.buffersRT[i+1];
rt_out = (dnnType *)netRT.buffersRT[i+out_count];
// there is the maxpool. It isn't an output but it is necessary for the process section
if(i==0)
out_count ++;
std::cout << "CUDNN vs correct";
checkResult(odim, cudnn_out, out);
@@ -391,5 +451,636 @@ int main()
std::cout << "CUDNN vs TRT ";
checkResult(odim, cudnn_out, rt_out);
}
TIMER_START
// -------- transofrm compose
cv::Mat imageOrig = cv::imread("/media/davide/DATA/shared_home/Projects/Professionale/repos/photo_2020-01-14_09-56-07.jpg");
cv::Mat imageF;
imageOrig.convertTo(imageF, CV_32FC3, 1/255.0);
cv::Mat image;
cv::Size sz = imageF.size();
std::cout<<"image: "<<sz.width<<", "<<sz.height<<std::endl;
resize(imageF, image, cv::Size(256, 256));
const int cropSize = 224;
const int offsetW = (image.cols - cropSize) / 2;
const int offsetH = (image.rows - cropSize) / 2;
const cv::Rect roi(offsetW, offsetH, cropSize, cropSize);
image = image(roi).clone();
std::cout << "Cropped image dimension: " << image.cols << " X " << image.rows << std::endl;
cv::Scalar mean_;
mean_ << 0.485, 0.456, 0.406;
cv::Scalar stddev_;
stddev_ << 0.229, 0.224, 0.225;
cv::Size s_im = imageF.size();
std::cout<<"size: "<<s_im.height<<" "<<s_im.width<<" - "<<std::endl;
std::cout<<"mean: "<<mean_<<", std: "<<stddev_<<std::endl;
cv::add(image, -mean_, image);
cv::divide(image, stddev_, image);
cv::Mat bgr[3];
dnnType *input, *input_d;
dim2 = dim;
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*dim2.tot()));
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim2.tot()));
image.convertTo(image, CV_32FC3, 1/255.0);
//split channels
cv::split(image,bgr);//split source
//write channels
for(int i=0; i<dim2.c; i++) {
int idx = i*image.rows*image.cols;
int ch = dim2.c-1 -i;
memcpy((void*)&input[idx], (void*)bgr[ch].data, image.rows*image.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
printCenteredTitle(" TENSORRT inference ", '=', 30); {
dim2.print();
TIMER_START
netRT.infer(dim2, input_d);
TIMER_STOP
dim2.print();
}
checkResult(dim2.tot(), input_h, input);
checkCuda(cudaFree(input_d));
checkCuda(cudaFreeHost(input));
// -----------------------------------pre-process ------------------------------------------
// it will resize the images to `512 x 512` in GETTING_STARTED.md
float scale = 1.0;
float new_height = sz.height * scale;
float new_width = sz.width * scale;
float inp_height = 224;//512;
float inp_width = 224;//512;
float c[] = {new_width / 2.0, new_height /2.0};
float s[2];
if(sz.width > sz.height){
s[0] = sz.width * 1.0;
s[1] = sz.width * 1.0;
}
else{
s[0] = sz.height * 1.0;
s[1] = sz.height * 1.0;
}
std::cout<<" "<<new_height<<" "<<new_width<<" "<<s[0]<<"-"<<s[1]<<" "<<c[0]<<"-"<<c[1]<<std::endl;
// ----------- get_affine_transform
// rot_rad = pi * 0 / 100 --> 0
cv::Mat src(cv::Size(2,3), CV_32F);
cv::Mat dst(cv::Size(2,3), CV_32F);
src.at<float>(0,0)=c[0];
src.at<float>(0,1)=c[1];
src.at<float>(1,0)=c[0];
src.at<float>(1,1)=c[1] + s[0] * -0.5;
dst.at<float>(0,0)=inp_width * 0.5;
dst.at<float>(0,1)=inp_height * 0.5;
dst.at<float>(1,0)=inp_width * 0.5;
dst.at<float>(1,1)=inp_height * 0.5 + inp_width * -0.5;
src.at<float>(2,0)=src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
src.at<float>(2,1)=src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
dst.at<float>(2,0)=dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
dst.at<float>(2,1)=dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
std::cout<<"src: "<<src<<std::endl;
std::cout<<"dst: "<<dst<<std::endl;
cv::Mat trans = cv::getAffineTransform( src, dst );
resize(imageOrig, image, cv::Size(new_width, new_height));
s_im = image.size();
std::cout<<"size: "<<s_im.height<<" "<<s_im.width<<" - "<<std::endl;
// image.convertTo(image, CV_32FC3, 1/255.0);
cv::warpAffine(image, image, trans, cv::Size(inp_width, inp_height), cv::INTER_LINEAR );
// cv::Scalar mean, stddev;
s_im = image.size();
std::cout<<"size: "<<s_im.height<<" "<<s_im.width<<" - "<<std::endl;
// for(int i=0; i<dim.tot(); i++ ){
// std::cout<<(float)image.at<cv::Vec3b>(0,i)[0]<<" - "<<(float)image.at<cv::Vec3b>(0,i)[1]<<" - "<<(float)image.at<cv::Vec3b>(0,i)[2]<<" - "<<std::endl;
// if(i==10)
// break;
// }
// return 0;
/////////////////////////////// ok fin qui
// cv::meanStdDev(image, mean, stddev );
// cv::Scalar mean(0.408, 0.447, 0.47);
cv::Vec<float, 3> mean;
mean << 0.408, 0.447, 0.47;
// s_im = mean.size();
// std::cout<<"size: "<<s_im.height<<" "<<s_im.width<<" - "<<std::endl;
cv::Vec<float, 3> stddev;
stddev << 0.289, 0.274, 0.278;
cv::Size s_imag = image.size();
std::cout<<"size: "<<s_imag.height<<" "<<s_imag.width<<" - "<<std::endl;
image.convertTo(image, CV_32FC3, 1/255.0);
std::cout<<"mean: "<<mean<<", std: "<<stddev<<std::endl;
// cv::add(image, -mean, image);
// cv::divide(image, stddev, image);
cv::MatIterator_<cv::Vec<float, 3>> it;
for(it = image.begin<cv::Vec<float, 3>>(); it != image.end<cv::Vec<float, 3>>(); ++it)
{
(*it)[0] = (float)(*it)[0] - mean[0];
(*it)[1] = (float)(*it)[1] - mean[1];
(*it)[2] = (float)(*it)[2] - mean[2];
(*it)[0] = (float)(*it)[0] / stddev[0];
(*it)[1] = (float)(*it)[1] / stddev[1];
(*it)[2] = (float)(*it)[2] / stddev[2];
}
// for(int i=0; i<dim.tot(); i++ ){
// std::cout<<(float)image.at<cv::Vec<float, 3>>(0,i)[0]<<" - "<<(float)image.at<cv::Vec<float, 3>>(0,i)[1]<<" - "<<(float)image.at<cv::Vec<float, 3>>(0,i)[2]<<" - "<<std::endl;
// if(i==10)
// break;
// }
// return 0;
/////////////////////// ok fin qui
std::cout<<"size: "<<s_imag.height<<" "<<s_imag.width<<" - "<<std::endl;
cv::Mat bgr2[3];
dnnType *input2, *input_d2;
dim2 = dim;
checkCuda(cudaMalloc(&input_d2, sizeof(dnnType)*dim2.tot()));
checkCuda(cudaMallocHost(&input2, sizeof(dnnType)*dim2.tot()));
// image.convertTo(image, CV_32FC3, 1/255.0);
//split channels
cv::split(image,bgr2);//split source
// std::cout<<"ch:\n";
// for(int i=0; i<dim.tot(); i++ ){
// std::cout<<(float)bgr2[0].at<cv::Vec<float, 3>>(0,i)[0]<<" - "<<(float)bgr2[0].at<cv::Vec<float, 3>>(0,i)[1]<<" - "<<(float)bgr2[0].at<cv::Vec<float, 3>>(0,i)[2]<<std::endl;
// if(i==10)
// break;
// }
// std::cout<<"ch:\n";
// for(int i=0; i<dim.tot(); i++ ){
// std::cout<<(float)bgr2[1].at<float>(1,i)<<std::endl;
// if(i==10)
// break;
// }
// std::cout<<"ch:\n";
// for(int i=0; i<dim.tot(); i++ ){
// std::cout<<(float)bgr2[2].at<float>(2,i)<<std::endl;
// if(i==10)
// break;
// }
// return 0;
///////////////////// ok fin qui
std::cout<<"\n\n\ncome: \n"<<image.rows<<" - "<<image.cols<<std::endl;
std::cout<<"reprint shape dim2\n";
dim2.print();
std::cout<<std::endl;
//write channels
for(int i=0; i<dim2.c; i++) {
int idx = i*image.rows*image.cols;
int ch = dim2.c-3 +i;
std::cout<<"i: "<<i<<", idx: "<<idx<<", ch: "<<ch<<std::endl;
memcpy((void*)&input2[idx], (void*)bgr2[ch].data, image.rows*image.cols*sizeof(dnnType));
}
// int k100 = 0;
// for(int i=1; i<=image.rows*image.cols*dim2.c; i++ ){
// std::cout<<input2[i]<<" ";
// if (i % (image.rows*image.cols) == 0){
// std::cout<<"\n\n";
// k100 ++;
// }
// }
// std::cout<<std::endl;
// std::cout<<"ci sono "<<k100<<" r\n";
// return 0;
///////////////////// pseudo ok
checkCuda(cudaMemcpyAsync(input_d2, input2, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
printCenteredTitle(" TENSORRT inference ", '=', 30); {
dim2.print();
TIMER_START
netRT.infer(dim2, input_d2);
TIMER_STOP
dim2.print();
}
checkResult(dim2.tot(), input_h, input2);
for(int i=0; i<dim.tot(); i++ ){
std::cout<<input_h[i]<<" "<<input2[i]<<std::endl;
if(i==10)
break;
}
// for(int i=0; i<dim.tot(); i++ ){
// std::cout<<input2[i]<<" ";
// }
// std::cout<<std::endl;
// return 0;
checkCuda(cudaFree(input_d2));
checkCuda(cudaFreeHost(input2));
// ------------------------------------ process --------------------------------------------
dnnType *hm_h;
checkCuda( cudaMallocHost(&hm_h, hm->output_dim.tot()*sizeof(dnnType)) );
dnnType *rt_out[4];
rt_out[0] = (dnnType *)netRT.buffersRT[1];
rt_out[1] = (dnnType *)netRT.buffersRT[2];
rt_out[2] = (dnnType *)netRT.buffersRT[3];
rt_out[3] = (dnnType *)netRT.buffersRT[4];
// checkCuda( cudaMemcpy(hm_h, rt_out[0], hm->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) );
std::cout<<"hm\n";
hm->output_dim.print();
// for(int i=0; i<hm->output_dim.tot(); i++ ){
// std::cout<<hm_h[i]<<" ";
// if(i==100)
// break;
// }
// std::cout<<"\n";
activationSIGMOIDForward(rt_out[0], rt_out[0], hm->output_dim.tot());
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(hm_h, rt_out[0], hm->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) );
std::cout<<"hm\n";
hm->output_dim.print();
// for(int i=0; i<hm->output_dim.tot(); i++ ){
// std::cout<<hm_h[i]<<" ";
// if(i==100)
// break;
// }
// std::cout<<"\n";
// return 0;
//////////////////////////// ok
// ----------- ctdet_decode
// perform nms on heatmaps
// tk::dnn::Layer *route_hm_layers[1] = { hm };
// tk::dnn::Route *route_hm = new tk::dnn::Route(&net, route_hm_layers, 1);
// ----------- nms
// int kernel = 3;
// int pad = (kernel - 1)/2;
// tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX);
// hmax_d = hmax->infer(hmax->input_dim.tot(), rt_out[0]);
// keep = (hmax == heat).float()
// return heat * keep
dnnType *hmax_h;
checkCuda( cudaMallocHost(&hmax_h, hmax->output_dim.tot()*sizeof(dnnType)) );
checkCuda( cudaMemcpy(hmax_h, rt_out[1], hmax->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) );
std::cout<<"hmax\n";
hmax->output_dim.print();
// for(int i=0; i<hmax->output_dim.tot(); i++ ){
// std::cout<<hmax_h[i]<<" ";
// if(i==100)
// break;
// }
// std::cout<<"\n";
// return 0;
// hm = hm * ( hmax == hm );
std::cout<<"hm:\n";
hm->output_dim.print();
std::cout<<"hmax:\n";
hmax->output_dim.print();
// return 0;
float toll = 0.000001;
for(int i=0; i < hm->output_dim.tot(); i++){
if(hm_h[i]-hmax_h[i] > toll || hm_h[i]-hmax_h[i] < -toll){
hm_h[i] = 0.0f;
}
}
// std::cout<<"\n";
// for(int i=0; i<hm->output_dim.tot(); i++ ){
// std::cout<<hm_h[i]<<" ";
// if(i==100)
// break;
// }
// std::cout<<"\n";
// return 0;
// checkCuda( cudaMemcpy(hm->dstData, hm_h, hm->output_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice) );
checkCuda( cudaFreeHost(hmax_h) );
// ----------- nms end
// ----------- topk
int K = 100;
int width = 56; // TODO
float *topk_scores;
int *topk_inds_;
float *topk_ys_;
float *topk_xs_;
std::cout<<"mah: "<<hm->output_dim.c * K<<std::endl;
checkCuda( cudaMallocHost(&topk_scores, hm->output_dim.c * K *sizeof(float)) );
checkCuda( cudaMallocHost(&topk_inds_, hm->output_dim.c * K *sizeof(int)) );
checkCuda( cudaMallocHost(&topk_ys_, hm->output_dim.c * K *sizeof(float)) );
checkCuda( cudaMallocHost(&topk_xs_, hm->output_dim.c * K *sizeof(float)) );
std::cout<<"1\n";
dnnType *hm_aus;
checkCuda( cudaMallocHost(&hm_aus, hm->output_dim.h * hm->output_dim.w *sizeof(dnnType)) );
std::cout<<"2\n";
int count;
std::vector<float> v = {2.0, 3.0, 9.0};
for (auto i: sort_indexes(v)) {
std::cout << i<< "--" <<v[i] << std::endl;
}
for(int i=0; i<hm->output_dim.c; i++){
count = 0;
// get the hm->output_dim.h * hm->output_dim.w elements for each channel and sort it. Then find the first 100 elements
checkCuda( cudaMemcpy(hm_aus, hm_h + i * hm->output_dim.h * hm->output_dim.w,
hm->output_dim.h * hm->output_dim.w * sizeof(dnnType), cudaMemcpyHostToHost) );
// std::cout<<"top scores: "<<hm->output_dim.h * hm->output_dim.w<<"\n";
// for(int k=0; k<hm->output_dim.h * hm->output_dim.w; k++)
// std::cout<<hm_aus[k]<<" ";
// std::cout<<std::endl;
// std::vector<float> my_vector {arr, arr + arr_length}
std::vector<float> my_vector{hm_aus, hm_aus + hm->output_dim.h * hm->output_dim.w};
for (auto j: sort_indexes(my_vector)) {
// std::cout <<"j: "<<j<<" -> "<< hm_aus[j] << std::endl;
topk_scores[i*K + count] = hm_aus[j];
topk_inds_[i*K +count] = j;
topk_ys_[i*K +count] = (int)(j / width);
topk_xs_[i*K +count] = (int)(j % width);
if(++count == K)
break;
}
}
std::cout<<"topk_xs_[0]: "<<topk_xs_[0]<<std::endl;
for(int i = 0; i< hm->output_dim.c * K; i++)
std::cout<<topk_xs_[i]<<" ";
std::cout<<"\n3\n";
// final
float *scores;
int *clses;
int *topk_inds;
float *topk_ys;
float *topk_xs;
checkCuda( cudaMallocHost(&scores, K *sizeof(float)) );
checkCuda( cudaMallocHost(&clses, K *sizeof(int)) );
checkCuda( cudaMallocHost(&topk_inds, K *sizeof(int)) );
checkCuda( cudaMallocHost(&topk_ys, K *sizeof(float)) );
checkCuda( cudaMallocHost(&topk_xs, K *sizeof(float)) );
std::cout<<"4\n";
count = 0;
std::vector<float> my_vector{topk_scores, topk_scores + hm->output_dim.c * K };
for (auto j: sort_indexes(my_vector)) {
// std::cout <<"j: "<<j<<" -> "<< hm_aus[j] << std::endl;
scores[count] = topk_scores[j];
clses[count] = (int)(j / K);
topk_inds[count] = topk_inds_[j];
topk_ys[count] = topk_ys_[j];
topk_xs[count] = topk_xs_[j];
if(++count == K)
break;
}
checkCuda( cudaFreeHost(topk_scores) );
checkCuda( cudaFreeHost(topk_inds_) );
checkCuda( cudaFreeHost(topk_ys_) );
checkCuda( cudaFreeHost(topk_xs_) );
std::cout<<"5\n";
// ----------- topk end
std::cout<<"topk_xs[0]: "<<topk_xs[0]<<std::endl;
for(int i = 0; i< K; i++)
std::cout<<topk_xs[i]<<" ";
std::cout<<std::endl;
/////////////////////////////// fin qui ok
dnnType *reg_aus;
checkCuda( cudaMallocHost(&reg_aus, reg->output_dim.tot()*sizeof(dnnType)) );
checkCuda( cudaMemcpy(reg_aus, rt_out[3], reg->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) );
std::cout<<"reg:\n";
reg->output_dim.print();
// for(int i=0; i<reg->output_dim.tot(); i++ ){
// std::cout<<reg_aus[i]<<" ";
// if(i==100)
// break;
// }
// std::cout<<"\n";
// return 0;
/////////////// ok
// for(int i=0; i<K; i++ ){
// std::cout<<reg_aus[topk_inds[i]]<<" "<<reg_aus[topk_inds[i]+56*56]<<std::endl;
// }
// std::cout<<"\n";
// return 0;
///////////////////// ok fin qui
for(int i = 0; i < K; i++){
topk_xs[i] = topk_xs[i] + reg_aus[topk_inds[i]];
topk_ys[i] = topk_ys[i] + reg_aus[topk_inds[i]+reg->output_dim.h*reg->output_dim.w];
}
std::cout<<"topk_xs[0]: "<<topk_xs[0]<<std::endl;
checkCuda( cudaFreeHost(reg_aus) );
std::cout<<"6\n";
dnnType *wh_aus;
float *bboxes;
checkCuda( cudaMallocHost(&wh_aus, wh->output_dim.tot()*sizeof(dnnType)) );
checkCuda( cudaMallocHost(&bboxes, 4 * K *sizeof(dnnType)) );
checkCuda( cudaMemcpy(wh_aus, rt_out[2], wh->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) );
std::cout<<"7\n";
for(int i = 0; i< K; i++)
std::cout<<topk_xs[i]<<" ";
std::cout<<std::endl;
std::cout<<topk_xs[0]<<std::endl;
std::cout<<topk_inds[0]<<std::endl;
std::cout<<wh_aus[topk_inds[0]*2]<<std::endl;
for(int i = 0; i < K; i++){
bboxes[i * 4] = topk_xs[i] - wh_aus[topk_inds[i]] / 2;
bboxes[i * 4 + 1] = topk_ys[i] - wh_aus[topk_inds[i]+reg->output_dim.h*reg->output_dim.w] / 2;
bboxes[i * 4 + 2] = topk_xs[i] + wh_aus[topk_inds[i]] / 2;
bboxes[i * 4 + 3] = topk_ys[i] + wh_aus[topk_inds[i]+reg->output_dim.h*reg->output_dim.w] / 2;
}
////////////////// fin qui ok
checkCuda( cudaFreeHost(wh_aus) );
checkCuda( cudaFreeHost(topk_inds) );
checkCuda( cudaFreeHost(topk_ys) );
checkCuda( cudaFreeHost(topk_xs) );
std::cout<<"8\n";
float *detections;
std::cout<<"bboxes:\n";
for(int i = 0; i < K+1; i++){
std::cout<<bboxes[i]<<" ";
}
std::cout<<std::endl;
checkCuda( cudaMallocHost(&detections, 6 * K *sizeof(dnnType)) );
checkCuda( cudaMemcpy(detections, bboxes, 4 * K *sizeof(dnnType), cudaMemcpyHostToHost) );
checkCuda( cudaMemcpy(detections + 4 * K *sizeof(dnnType), scores, K *sizeof(dnnType), cudaMemcpyHostToHost) );
checkCuda( cudaMemcpy(detections + 5 * K *sizeof(dnnType), clses, K *sizeof(dnnType), cudaMemcpyHostToHost) );
checkCuda( cudaFreeHost(bboxes) );
// checkCuda( cudaFreeHost(scores) );
// checkCuda( cudaFreeHost(clses) );
// servono [bboxes, scores, clses]
checkCuda( cudaDeviceSynchronize() );
// ---------------------------------- post-process -----------------------------------------
// --------- ctdet_post_process
//for 1
// float *dets;
// checkCuda( cudaMallocHost(&dets, 2 * K *sizeof(float)) );
// checkCuda( cudaMemcpy(dets, detections, 2 * K *sizeof(float), cudaMemcpyHostToHost) );
// --------- transform_preds
float *target_coords;
checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) );
src.at<float>(0,0)=c[0];
src.at<float>(0,1)=c[1];
src.at<float>(1,0)=c[0];
src.at<float>(1,1)=c[1] + s[0] * -0.5;
dst.at<float>(0,0)=width * 0.5;
dst.at<float>(0,1)=width * 0.5;
dst.at<float>(1,0)=width * 0.5;
dst.at<float>(1,1)=width * 0.5 + width * -0.5;
src.at<float>(2,0)=src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
src.at<float>(2,1)=src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
dst.at<float>(2,0)=dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
dst.at<float>(2,1)=dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
std::cout<<"src: "<<src<<std::endl;
std::cout<<"dst: "<<dst<<std::endl;
cv::Size s_aus;
cv::Mat trans2(cv::Size(3,2), CV_32F);
trans2 = cv::getAffineTransform( dst, src );
s_aus = trans2.size();
std::cout<<"trnas2: "<<trans2<<std::endl;
std::cout<<trans2.at<double>(0,0)<<" - "<<trans2.at<double>(0,1)<<" - "<<trans2.at<double>(0,2)<<"\n"<<trans2.at<double>(1,0)<<" - "<<trans2.at<double>(1,1)<<" - "<<trans2.at<double>(1,2)<<std::endl;
std::cout<<"size: "<<s_aus.height<<" "<<s_aus.width<<" - "<<std::endl;
cv::Mat new_pt1(cv::Size(1,2), CV_32F);
cv::Mat new_pt2(cv::Size(1,2), CV_32F);
for(int i = 0; i<K; i++){
// new_pt1.at<float>(0,0)=detections[i*4];
// new_pt1.at<float>(0,1)=detections[i*4+1];
// new_pt1.at<float>(0,2)=1.0;
// new_pt1 << detections[i], detections[i+K], 1.0;
// std::cout<<"----\ni: "<<i<<std::endl;//<<" newpt: "<<new_pt1<<std::endl;
// std::cout<<"origi: "<<detections[i*4]<<", "<<detections[i*4+1]<<", "<<1.0<<std::endl;
s_aus = new_pt1.size();
// std::cout<<"size: "<<s_aus.height<<" "<<s_aus.width<<" - "<<std::endl;
// new_pt2 = trans2.dot(new_pt1);
new_pt1.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*detections[i*4] +
static_cast<float>(trans2.at<double>(0,1))*detections[i*4+1] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt1.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*detections[i*4] +
static_cast<float>(trans2.at<double>(1,1))*detections[i*4+1] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
new_pt2.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*detections[i*4+2] +
static_cast<float>(trans2.at<double>(0,1))*detections[i*4+3] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt2.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*detections[i*4+2] +
static_cast<float>(trans2.at<double>(1,1))*detections[i*4+3] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
// std::cout<<"\n new: "<<new_pt1<<" - "<<new_pt2<<std::endl;
target_coords[i*4] = new_pt1.at<float>(0,0);
target_coords[i*4+1] = new_pt1.at<float>(0,1);
target_coords[i*4+2] = new_pt2.at<float>(0,0);
target_coords[i*4+3] = new_pt2.at<float>(0,1);
// std::cout<<new_pt1.at<float>(0,0)<<", "<<new_pt1.at<float>(0,1)<<", "<<new_pt2.at<float>(0,0)<<", "<<new_pt2.at<float>(0,1)<<std::endl;
}
// return 0;
// /////////////// ok fin qui
const char *coco_class_name_ [] = {"person", "bicycle", "car", "motorcycle", "airplane",
"bus", "train", "truck", "boat", "traffic light", "fire hydrant",
"stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse",
"sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
"umbrella", "handbag", "tie", "suitcase", "frisbee", "skis",
"snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
"skateboard", "surfboard", "tennis racket", "bottle", "wine glass",
"cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich",
"orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake",
"chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv",
"laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
"oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
"scissors", "teddy bear", "hair drier", "toothbrush"};
std::vector<std::string> coco_class_name(coco_class_name_, std::end( coco_class_name_ ));
int num_classes = 80;
float vis_threshold = 0.3;
// int *classes;
std::vector<tk::dnn::box> detected;
// checkCuda( cudaMallocHost(&classes, K *sizeof(int)) );
// checkCuda( cudaMemcpy(classes, detections + 5 * K *sizeof(dnnType), K *sizeof(dnnType), cudaMemcpyHostToHost) );
for(int i = 0; i<num_classes; i++){
for(int j=0; j<K; j++)
if(clses[j] == i){
//TODO recupera detections[j +0 +1 +2 +3 +4(+5 è la classes, già presa)]
// queste compongono un ogg assegnato alla classe i (0, 79) --> i+1 (1:80);
if(scores[j] > vis_threshold){
std::cout<<"th: "<<scores[j]<<" - cl: "<<clses[j]<<" i: "<<i<<std::endl;
//add coco bbox
//det[0:4], i, det[4]
int x0 = target_coords[j*4];
int y0 = target_coords[j*4+1];
int x1 = target_coords[j*4+2];
int y1 = target_coords[j*4+3];
int obj_class = clses[j];
float prob = scores[j];
std::cout<<"("<<x0<<", "<<y0<<"),("<<x1<<", "<<y1<<")"<<std::endl;
tk::dnn::box res;
res.cl = obj_class;
res.prob = prob;
res.x = x0;
res.y = y0;
res.w = x1 - x0;
res.h = y1 - y0;
detected.push_back(res);
}
}
}
tk::dnn::box b;
int x0, w, x1, y0, h, y1;
int objClass;
std::string det_class;
;
int baseline = 0;
float fontScale = 0.5;
int thickness = 2;
cv::Scalar colors[256];
for(int c=0; c<num_classes; c++) {
int offset = c*123457 % num_classes;
float r = get_color(2, offset, num_classes);
float g = get_color(1, offset, num_classes);
float b = get_color(0, offset, num_classes);
colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
}
int num_detected = detected.size();
for (int i = 0; i < num_detected; i++){
b = detected[i];
x0 = b.x;
w = b.w;
x1 = b.x + w;
y0 = b.y;
h = b.h;
y1 = b.y + h;
objClass = b.cl;
det_class = coco_class_name[objClass];
cv::rectangle(imageOrig, cv::Point(x0, y0), cv::Point(x1, y1), colors[objClass], 2);
// draw label
cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline);
cv::rectangle(imageOrig, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), colors[b.cl], -1);
cv::putText(imageOrig, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness);
}
cv::namedWindow("cnet", cv::WINDOW_NORMAL);
cv::imshow("cnet", imageOrig);
cv::waitKey(10000);
TIMER_STOP
return 0;
}