Merge branch 'ceccocats:master' into master
This commit is contained in:
Harshvardhan Chandirasekar
GitHub
@@ -119,6 +119,16 @@ target_link_libraries(test_resnet101_cnet tkDNN)
|
||||
add_executable(test_dla34_cnet tests/centernet/dla34_cnet/dla34_cnet.cpp)
|
||||
target_link_libraries(test_dla34_cnet tkDNN)
|
||||
|
||||
# SHELFNET
|
||||
add_executable(test_shelfnet tests/shelfnet/shelfnet.cpp)
|
||||
target_link_libraries(test_shelfnet tkDNN)
|
||||
|
||||
add_executable(test_shelfnet_berkeley tests/shelfnet/shelfnet_berkeley.cpp)
|
||||
target_link_libraries(test_shelfnet_berkeley tkDNN)
|
||||
|
||||
add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp)
|
||||
target_link_libraries(test_shelfnet_mapillary tkDNN)
|
||||
|
||||
# DEMOS
|
||||
add_executable(test_rtinference tests/test_rtinference/rtinference.cpp)
|
||||
target_link_libraries(test_rtinference tkDNN)
|
||||
@@ -129,6 +139,9 @@ target_link_libraries(map_demo tkDNN)
|
||||
add_executable(demo demo/demo/demo.cpp)
|
||||
target_link_libraries(demo tkDNN)
|
||||
|
||||
add_executable(seg_demo demo/demo/seg_demo.cpp)
|
||||
target_link_libraries(seg_demo tkDNN)
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Install
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
@@ -17,6 +17,10 @@ If you use tkDNN in your research, please cite the [following paper](https://iee
|
||||
}
|
||||
```
|
||||
|
||||
### What's new (20 July 2021)
|
||||
- [x] Support to sematic segmentation [README](docs/README_seg.md)
|
||||
- [ ] Support to TensorRT8 (WIP)
|
||||
|
||||
## FPS Results
|
||||
Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on
|
||||
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
|
||||
@@ -89,16 +93,21 @@ Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001
|
||||
- [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Dependencies
|
||||
This branch works on every NVIDIA GPU that supports the dependencies:
|
||||
* CUDA 10.0
|
||||
* CUDNN 7.603
|
||||
* TENSORRT 6.01
|
||||
* OPENCV 3.4
|
||||
* yaml-cpp 0.5.2 (sudo apt install libyaml-cpp-dev)
|
||||
This branch works on every NVIDIA GPU that supports the following (latest tested) dependencies:
|
||||
* CUDA 11.0 (or >= 10)
|
||||
* cuDNN 8.0.4 (or >= 7.3)
|
||||
* TensorRT 7.2.0 (or >=5)
|
||||
* OpenCV 4.5.2 (or >=4)
|
||||
* cmake 3.21 (or >= 3.15)
|
||||
* yaml-cpp 0.5.2
|
||||
* eigen3 3.3.4
|
||||
* curl 7.58
|
||||
|
||||
```
|
||||
sudo apt install libyaml-cpp-dev curl libeigen3-dev
|
||||
|
||||
```
|
||||
|
||||
## About OpenCV
|
||||
To compile and install OpenCV4 with contrib us the script ```install_OpenCV4.sh```. It will download and compile OpenCV in Download folder.
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
#include <iostream>
|
||||
#include <signal.h>
|
||||
#include <stdlib.h> /* srand, rand */
|
||||
#include <unistd.h>
|
||||
#include <mutex>
|
||||
|
||||
#include "SegmentationNN.h"
|
||||
|
||||
bool gRun;
|
||||
bool SAVE_RESULT = true;
|
||||
|
||||
void sig_handler(int signo) {
|
||||
std::cout<<"request gateway stop\n";
|
||||
gRun = false;
|
||||
}
|
||||
|
||||
void writePred(const std::string& images_names, const std::string& gt_folder, const std::string& out_folder, tk::dnn::SegmentationNN& segNN, int& width, int& height, bool show=false){
|
||||
std::ifstream all_gt(images_names);
|
||||
std::string filename;
|
||||
cv::Mat frame;
|
||||
for (; std::getline(all_gt, filename); ) {
|
||||
std::cout<<filename<<std::endl;
|
||||
frame = cv::imread(gt_folder + filename);
|
||||
height = frame.rows;
|
||||
width = frame.cols;
|
||||
segNN.updateOriginal(frame, false);
|
||||
if(show)
|
||||
segNN.draw();
|
||||
cv::imwrite(out_folder + filename, segNN.segmented[0]);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
|
||||
std::cout<<"detection\n";
|
||||
signal(SIGINT, sig_handler);
|
||||
|
||||
|
||||
std::string net = "shelfnet_fp32.rt";
|
||||
if(argc > 1)
|
||||
net = argv[1];
|
||||
std::string input = "../demo/yolo_test.mp4";
|
||||
if(argc > 2)
|
||||
input = argv[2];
|
||||
int n_batch = 1;
|
||||
if(argc > 3)
|
||||
n_batch = atoi(argv[3]);
|
||||
int n_classes = 19;
|
||||
if(argc > 4)
|
||||
n_classes = atoi(argv[4]);
|
||||
bool resize = false;
|
||||
if(argc > 5)
|
||||
resize = atoi(argv[5]);
|
||||
int baseline_resize = 1024;
|
||||
if(argc > 6)
|
||||
baseline_resize = atoi(argv[6]);
|
||||
bool show = true;
|
||||
if(argc > 7)
|
||||
show = atoi(argv[7]);
|
||||
bool write_pred = false;
|
||||
if(argc > 8)
|
||||
write_pred = atoi(argv[8]);
|
||||
|
||||
if(resize && (baseline_resize < 0 || baseline_resize > 5000))
|
||||
FatalError("Problem with baseline resize")
|
||||
if(n_batch < 1 || n_batch > 64)
|
||||
FatalError("Batch dim not supported");
|
||||
|
||||
//net initialization
|
||||
tk::dnn::SegmentationNN segNN;
|
||||
segNN.init(net, n_classes, n_batch);
|
||||
|
||||
int height = 0, width = 0;
|
||||
int basewidth=baseline_resize, hsize;
|
||||
|
||||
if(write_pred){
|
||||
std::string gt_folder = "../demo/CityScapes_val/images/";
|
||||
std::string images_names = "../demo/CityScapes_val/all_images.txt";
|
||||
std::string out_folder = "seg/";
|
||||
|
||||
writePred(images_names, gt_folder, out_folder, segNN, width, height, show);
|
||||
}
|
||||
else{
|
||||
if(!show)
|
||||
SAVE_RESULT = true;
|
||||
|
||||
gRun = true;
|
||||
|
||||
cv::VideoCapture cap(input);
|
||||
if(!cap.isOpened())
|
||||
gRun = false;
|
||||
else
|
||||
std::cout<<"camera started\n";
|
||||
|
||||
cv::VideoWriter resultVideo;
|
||||
if(SAVE_RESULT) {
|
||||
int w,h;
|
||||
if(resize){
|
||||
w = basewidth;
|
||||
h = int((float(cap.get(cv::CAP_PROP_FRAME_HEIGHT))*float(basewidth/float(cap.get(cv::CAP_PROP_FRAME_WIDTH)))));
|
||||
}
|
||||
else{
|
||||
w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
|
||||
h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
|
||||
}
|
||||
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
|
||||
}
|
||||
|
||||
cv::Mat frame;
|
||||
while(gRun) {
|
||||
cap >> frame;
|
||||
if(!frame.data)
|
||||
break;
|
||||
|
||||
if(resize){
|
||||
hsize = int((float(frame.rows)*float(basewidth/float(frame.cols))));
|
||||
cv::resize(frame, frame, cv::Size(basewidth, hsize));
|
||||
}
|
||||
|
||||
height = frame.rows;
|
||||
width = frame.cols;
|
||||
|
||||
//inference
|
||||
segNN.updateOriginal(frame, true);
|
||||
if(show)
|
||||
segNN.draw();
|
||||
|
||||
if(SAVE_RESULT)
|
||||
resultVideo << segNN.segmented[0];
|
||||
}
|
||||
}
|
||||
|
||||
std::cout<<"segmentation end\n";
|
||||
double mean = 0, mean_pre = 0, mean_post = 0;
|
||||
|
||||
std::cout<<COL_GREENB<<"\n\nTime stats for size ["<<width<<","<<height<<"] :\n";
|
||||
|
||||
for(int i=0; i<segNN.stats.size(); i++) mean += segNN.stats[i]; mean /= segNN.stats.size();
|
||||
for(int i=0; i<segNN.stats_pre.size(); i++) mean_pre += segNN.stats_pre[i]; mean_pre /= segNN.stats_pre.size();
|
||||
for(int i=0; i<segNN.stats_post.size(); i++) mean_post += segNN.stats_post[i]; mean_post /= segNN.stats_post.size();
|
||||
std::cout<<"Avg pre:\t"<<mean_pre<<" ms\t"<<1000/(mean_pre)<<" FPS\n";
|
||||
std::cout<<"Avg inf:\t"<<mean<<" ms\t"<<1000/(mean)<<" FPS\n";
|
||||
std::cout<<"Avg post:\t"<<mean_post<<" ms\t"<<1000/(mean_post)<<" FPS\n\n";
|
||||
std::cout<<"Avg tot:\t"<<(mean_pre + mean_post + mean) <<" ms\t"<<1000/((mean_pre + mean_post + mean))<<" FPS\n"<<COL_END;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# Semantic Segmentation with tkDNN
|
||||
|
||||
Currently tkDNN supports only ShelfNet as semantic segmentation network.
|
||||
|
||||
## Export weights from Shelfnet
|
||||
To get the weights needed to run Shelfnet tests use [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of a Pytorch implementation of Shelfnet network.
|
||||
|
||||
```
|
||||
git clone https://git.hipert.unimore.it/mverucchi/shelfnet
|
||||
cd shelfnet
|
||||
cd ShelfNet18_realtime
|
||||
conda env create --file shelfnet_env.yml
|
||||
conda activate shelfnet
|
||||
mkdir layer debug
|
||||
python export.py
|
||||
```
|
||||
|
||||
|
||||
## Run the demo
|
||||
|
||||
To run the semantic segmentation demo follow these steps (example with shelfnet):
|
||||
```
|
||||
rm shelfnet_fp32.rt # be sure to delete(or move) old tensorRT files
|
||||
export TKDNN_BATCHSIZE=4 # be sure you have batch size > than 1 if you want to run inference on images bigger than 1024
|
||||
./test_shelfnet # run the yolo test (is slow)
|
||||
./demo shelfnet_fp32.rt ../demo/yolo_test.mp4 1 19
|
||||
```
|
||||
In general the demo program takes the following parameters:
|
||||
```
|
||||
./seg_demo <network-rt-file> <path-to-video> <n-batches> <number-of-classes> <resize-flag> <baseline-resize> <show-flag> <write-pred>
|
||||
```
|
||||
where
|
||||
* ```<network-rt-file>``` is the rt file generated by a test
|
||||
* ```<<path-to-video>``` is the path to a video file or a camera input
|
||||
* ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
|
||||
* ```<number-of-classes>```is the number of classes the network is trained on
|
||||
* ```<resize-flag>``` if set to 0 the demo will not resize the input frames, but use it as it is, otherwise it will resize it.
|
||||
* ```<baseline-resize>``` is ```<resize-flag>``` is set to 1, then the input frames will be proportionally resized using ```<baseline-resize>``` as width baseline.
|
||||
* ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
|
||||
* ```<write-pred>``` if set to 0 (default) the demo will run, otherwise the evaluation of a dataset will run and the output of the segmentation will be saved. Attention: this is under development and paths are embedded, so change them in the code in advance.
|
||||
|
||||
NB) By default it is used FP32 inference
|
||||
NB) The batching is not used to work on more streams, rather to work on more tiles of the same image. Shelfnet never resized the input image, therefore for images greater than 1024x1024 tiles of 1024x1024 are given in input to the network in batch.
|
||||
|
||||
|
||||
|
||||
For other demo videos refer to [this playlist](https://www.youtube.com/playlist?list=PLv0nEQYDD45y5EdSiywwCGPBmJVUzIWwe).
|
||||
|
||||
|
||||
## Existing tests and supported networks
|
||||
|
||||
| Test Name | Network | Dataset | N Classes | Input size | Weights |
|
||||
| :---------------- | :-------------------------------------------- | :-----------------------------------------------------------: | :-------: | :-----------: | :------------------------------------------------------------------------ |
|
||||
| shelfnet | ShelfNet18_realtime<sup>1</sup> | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) |
|
||||
| shelfnet_berkeley | ShelfNet18_realtime<sup>1</sup> | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) |
|
||||
|
||||
1. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019.
|
||||
|
||||
|
||||
## FPS Results
|
||||
|
||||
Inference FPS of shelfnet with tkDNN, average of 1200 images on:
|
||||
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
|
||||
* Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
|
||||
|
||||
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
|
||||
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
|
||||
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | pre | 6.11863 | 163.435 | 5.81465 | 171.979 | 5.88699 | 169.866 |
|
||||
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | inf | 11.5464 | 86.6074 | 7.35396 | 135.981 | 6.37623 | 156.832 |
|
||||
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | post | 4.09058 | 244.464 | 3.91961 | 255.128 | 4.07343 | 245.493 |
|
||||
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | tot | 21.7556 | 45.9652 | 17.0882 | 58.5199 | 16.3366 | 61.2121 |
|
||||
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | pre | 25.435 | 39.3158 | 25.2953 | 39.5331 | 25.9303 | 38.565 |
|
||||
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | inf | 36.5015 | 27.3961 | 17.0534 | 58.6395 | 15.6061 | 64.0773 |
|
||||
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | post | 17.3917 | 57.4985 | 17.1649 | 58.2583 | 17.5539 | 56.9675 |
|
||||
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | tot | 79.3283 | 12.6058 | 59.5136 | 16.8029 | 59.0903 | 16.9233 |
|
||||
| AGX Xavier | shelfnet 1024x1024 (B=1) | pre | 8.0174 | 124.729 | 7.5117 | 133.126 | 7.47333 | 133.809 |
|
||||
| AGX Xavier | shelfnet 1024x1024 (B=1) | inf | 72.4173 | 13.8089 | 37.505 | 26.6631 | 31.3286 | 31.9197 |
|
||||
| AGX Xavier | shelfnet 1024x1024 (B=1) | post | 8.89958 | 112.365 | 8.83576 | 113.176 | 9.42655 | 106.083 |
|
||||
| AGX Xavier | shelfnet 1024x1024 (B=1) | tot | 89.3342 | 11.1939 | 53.8525 | 18.5692 | 48.2285 | 20.7346 |
|
||||
| AGX Xavier | shelfnet 2048x2048 (B=4) | pre | 47.1454 | 21.211 | 21.6475 | 46.1947 | 21.4201 | 46.6851 |
|
||||
| AGX Xavier | shelfnet 2048x2048 (B=4) | inf | 266.537 | 3.75183 | 128.321 | 7.79293 | 107.621 | 9.29185 |
|
||||
| AGX Xavier | shelfnet 2048x2048 (B=4) | post | 44.0711 | 22.6906 | 40.1732 | 24.8922 | 39.873 | 25.0796 |
|
||||
| AGX Xavier | shelfnet 2048x2048 (B=4) | tot | 357.753 | 2.79522 | 190.142 | 5.25922 | 168.914 | 5.92016 |
|
||||
|
||||
|
||||
## Known issues
|
||||
|
||||
When creating the rt file all the checks returns errors. It is due to a different resize function and handling of the original ShelfNet outputs.
|
||||
However, the network is supposed to work.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 6.3 MiB |
+23
-2
@@ -22,6 +22,7 @@ enum layerType_t {
|
||||
LAYER_ACTIVATION_LOGISTIC,
|
||||
LAYER_FLATTEN,
|
||||
LAYER_RESHAPE,
|
||||
LAYER_RESIZE,
|
||||
LAYER_MULADD,
|
||||
LAYER_POOLING,
|
||||
LAYER_SOFTMAX,
|
||||
@@ -72,6 +73,7 @@ public:
|
||||
case LAYER_ACTIVATION_LOGISTIC: return "ActivationLogistic";
|
||||
case LAYER_FLATTEN: return "Flatten";
|
||||
case LAYER_RESHAPE: return "Reshape";
|
||||
case LAYER_RESIZE: return "Resize";
|
||||
case LAYER_MULADD: return "MulAdd";
|
||||
case LAYER_POOLING: return "Pooling";
|
||||
case LAYER_SOFTMAX: return "Softmax";
|
||||
@@ -226,8 +228,9 @@ class Activation : public Layer {
|
||||
public:
|
||||
int act_mode;
|
||||
float ceiling;
|
||||
float slope;
|
||||
|
||||
Activation(Network *net, int act_mode, const float ceiling=0.0);
|
||||
Activation(Network *net, int act_mode, const float ceiling=0.0, const float slope=0.1);
|
||||
virtual ~Activation();
|
||||
virtual layerType_t getLayerType() {
|
||||
if(act_mode == CUDNN_ACTIVATION_CLIPPED_RELU)
|
||||
@@ -432,6 +435,23 @@ public:
|
||||
|
||||
};
|
||||
|
||||
enum ResizeMode_t { NEAREST= 0,
|
||||
LINEAR= 1};
|
||||
|
||||
/**
|
||||
Resize layer
|
||||
*/
|
||||
class Resize : public Layer {
|
||||
|
||||
public:
|
||||
Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed=false, ResizeMode_t mode=NEAREST);
|
||||
virtual ~Resize();
|
||||
virtual layerType_t getLayerType() { return LAYER_RESIZE; };
|
||||
|
||||
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
|
||||
|
||||
ResizeMode_t mode;
|
||||
};
|
||||
|
||||
/**
|
||||
MulAdd layer
|
||||
@@ -552,7 +572,7 @@ public:
|
||||
class Shortcut : public Layer {
|
||||
|
||||
public:
|
||||
Shortcut(Network *net, Layer *backLayer);
|
||||
Shortcut(Network *net, Layer *backLayer, bool mul=false);
|
||||
virtual ~Shortcut();
|
||||
virtual layerType_t getLayerType() { return LAYER_SHORTCUT; };
|
||||
|
||||
@@ -560,6 +580,7 @@ public:
|
||||
|
||||
public:
|
||||
Layer *backLayer;
|
||||
bool mul = false;
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -108,6 +108,7 @@ public:
|
||||
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
|
||||
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Flatten *l);
|
||||
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reshape *l);
|
||||
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Resize *l);
|
||||
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
|
||||
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
|
||||
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
cv::Mat vizFloat2colorMap(cv::Mat map);
|
||||
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim);
|
||||
cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, int classes=19);
|
||||
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=19);
|
||||
cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000);
|
||||
|
||||
}}
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
#ifndef SEGMENTATIONNN_H
|
||||
#define SEGMENTATIONNN_H
|
||||
|
||||
#include <iostream>
|
||||
#include <signal.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <mutex>
|
||||
#include "utils.h"
|
||||
|
||||
#include <opencv2/core/core.hpp>
|
||||
#include <opencv2/highgui/highgui.hpp>
|
||||
#include <opencv2/imgproc/imgproc.hpp>
|
||||
#include <opencv2/core/hal/interface.h>
|
||||
|
||||
#include "tkdnn.h"
|
||||
#include "NetworkViz.h"
|
||||
#include "kernelsThrust.h"
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
class SegmentationNN {
|
||||
|
||||
protected:
|
||||
tk::dnn::NetworkRT *netRT = nullptr;
|
||||
int nBatches = 1;
|
||||
|
||||
std::vector<cv::Size> originalSize;
|
||||
cv::Mat bgr[3];
|
||||
dnnType *input;
|
||||
dnnType *input_d;
|
||||
float* confidences_h;
|
||||
|
||||
float * tmpInputData_d;
|
||||
float *tmpOutData_d;
|
||||
float *tmpOutData_h;
|
||||
|
||||
float *mean_d, *stddev_d;
|
||||
|
||||
cublasHandle_t cublasHandle;
|
||||
|
||||
void computeBorders(const int or_width, const int or_height, int& top, int& bottom, int& left, int&right){
|
||||
top = 0;
|
||||
bottom = 0;
|
||||
left = 0;
|
||||
right = 0;
|
||||
|
||||
if(or_height != or_width){
|
||||
if(or_height < or_width){
|
||||
top = (or_width - or_height)/2;
|
||||
bottom = or_width - top - or_height;
|
||||
}
|
||||
else{
|
||||
left = (or_height - or_width)/2;
|
||||
right = or_height - left - or_width;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method preprocess the image, before feeding it to the NN.
|
||||
*
|
||||
* @param frame original frame to adapt for inference.
|
||||
* @param bi batch index
|
||||
*/
|
||||
void preprocess(cv::Mat &frame, const int bi=0) {
|
||||
originalSize[bi] = frame.size();
|
||||
|
||||
frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0);
|
||||
int H = frame.rows;
|
||||
int W = frame.cols;
|
||||
cv::Mat frame_cropped;
|
||||
|
||||
int top, bottom, left, right;
|
||||
computeBorders(W, H, top, bottom, left, right);
|
||||
cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) );
|
||||
|
||||
tk::dnn::dataDim_t idim = netRT->input_dim;
|
||||
|
||||
resize(frame_cropped, frame_cropped, cv::Size(idim.w, idim.h));
|
||||
|
||||
cv::split(frame_cropped, bgr);
|
||||
for (int i = 0; i < idim.c; i++){
|
||||
int idx = i * frame_cropped.rows * frame_cropped.cols;
|
||||
int ch = idim.c-1 -i;
|
||||
memcpy((void *)&input[idx + idim.tot()*bi], (void *)bgr[ch].data, frame_cropped.rows * frame_cropped.cols * sizeof(dnnType));
|
||||
}
|
||||
|
||||
checkCuda(cudaMemcpyAsync(input_d+ idim.tot()*bi, input + idim.tot()*bi, idim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
|
||||
|
||||
normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d);
|
||||
}
|
||||
|
||||
/**
|
||||
* This method postprocess the output of the NN to obtain the correct
|
||||
* boundig boxes.
|
||||
*
|
||||
* @param bi batch index
|
||||
*/
|
||||
void postprocess(const int bi=0, bool appy_colormap = true) {
|
||||
dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
|
||||
|
||||
dataDim_t odim = netRT->output_dim;
|
||||
|
||||
matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h);
|
||||
maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w);
|
||||
checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost));
|
||||
|
||||
dataDim_t vdim = odim;
|
||||
vdim.c = 1;
|
||||
|
||||
cv::Mat colored;
|
||||
|
||||
if(appy_colormap)
|
||||
colored = vizData2Mat(tmpOutData_h, vdim, netRT->input_dim.h, netRT->input_dim.w, 0, classes, classes);
|
||||
else{
|
||||
cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h);
|
||||
colored_fp32.convertTo(colored, CV_8UC1);
|
||||
}
|
||||
|
||||
int max_dim = (originalSize[bi].width > originalSize[bi].height) ? originalSize[bi].width : originalSize[bi].height;
|
||||
resize(colored, colored, cv::Size(max_dim, max_dim));
|
||||
int top, bottom, left, right;
|
||||
computeBorders(originalSize[bi].width, originalSize[bi].height, top, bottom, left, right);
|
||||
cv::Rect roi(left,top,originalSize[bi].width, originalSize[bi].height);
|
||||
cv::Mat or_size (colored, roi);
|
||||
segmented[bi] = or_size;
|
||||
};
|
||||
|
||||
public:
|
||||
int classes = 0;
|
||||
std::vector<double> stats; /*keeps track of inference times (ms)*/
|
||||
std::vector<double> stats_pre;
|
||||
std::vector<double> stats_post;
|
||||
std::vector<std::string> classesNames;
|
||||
std::vector<cv::Mat> segmented;
|
||||
|
||||
SegmentationNN() {
|
||||
checkERROR( cublasCreate(&cublasHandle) );
|
||||
};
|
||||
~SegmentationNN(){
|
||||
checkERROR( cublasDestroy(cublasHandle) );
|
||||
};
|
||||
|
||||
/**
|
||||
* Method used to inialize the class, allocate memory and compute
|
||||
* needed data.
|
||||
*
|
||||
* @param tensor_path path to the rt file og the NN.
|
||||
* @param n_classes number of classes for the given dataset.
|
||||
* @param n_batches maximum number of batches to use in inference
|
||||
* @return true if everything is correct, false otherwise.
|
||||
*/
|
||||
bool init(const std::string& tensor_path, const int n_classes=19, const int n_batches=1){
|
||||
std::cout<<(tensor_path).c_str()<<"\n";
|
||||
if(!fileExist(tensor_path.c_str()))
|
||||
FatalError("This file do not exists" + tensor_path );
|
||||
|
||||
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str());
|
||||
classes = n_classes;
|
||||
nBatches = n_batches;
|
||||
|
||||
checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
|
||||
checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
|
||||
|
||||
dataDim_t odim = netRT->output_dim;
|
||||
|
||||
checkCuda(cudaMallocHost(&confidences_h, sizeof(float) * odim.tot()));
|
||||
checkCuda(cudaMalloc(&tmpInputData_d, sizeof(float) * odim.tot()));
|
||||
checkCuda(cudaMalloc(&tmpOutData_d, sizeof(float) * odim.w*odim.h));
|
||||
checkCuda(cudaMallocHost(&tmpOutData_h, sizeof(float) * odim.w*odim.h));
|
||||
|
||||
segmented.resize(nBatches);
|
||||
originalSize.resize(nBatches);
|
||||
|
||||
std::vector<float> mean = {0.485, 0.456, 0.406};
|
||||
std::vector<float> stddev = {0.229, 0.224, 0.225};
|
||||
|
||||
checkCuda(cudaMalloc(&mean_d, sizeof(float) * mean.size()));
|
||||
checkCuda(cudaMalloc(&stddev_d, sizeof(float) * stddev.size()));
|
||||
|
||||
checkCuda(cudaMemcpyAsync(mean_d, mean.data(), mean.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream));
|
||||
checkCuda(cudaMemcpyAsync(stddev_d, stddev.data(), stddev.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream));
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* This method performs the whole detection of the NN.
|
||||
*
|
||||
* @param frames frames to run detection on.
|
||||
* @param cur_batches number of batches to use in inference
|
||||
* @param save_times if set to true, preprocess, inference and postprocess times
|
||||
* are saved on a csv file, otherwise not.
|
||||
* @param times pointer to the output stream where to write times
|
||||
* @param mAP set to true only if all the probabilities for a bounding
|
||||
* box are needed, as in some cases for the mAP calculation
|
||||
*/
|
||||
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool apply_colormap=true){
|
||||
if(cur_batches > nBatches)
|
||||
FatalError("A batch size greater than nBatches cannot be used");
|
||||
|
||||
originalSize.clear();
|
||||
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
|
||||
{
|
||||
TKDNN_TSTART
|
||||
for(int bi=0; bi<cur_batches;++bi){
|
||||
if(!frames[bi].data)
|
||||
FatalError("No image data feed to detection");
|
||||
originalSize.push_back(frames[bi].size());
|
||||
preprocess(frames[bi], bi);
|
||||
}
|
||||
TKDNN_TSTOP
|
||||
stats_pre.push_back(t_ns);
|
||||
}
|
||||
|
||||
//do inference
|
||||
tk::dnn::dataDim_t dim = netRT->input_dim;
|
||||
dim.n = cur_batches;
|
||||
{
|
||||
if(TKDNN_VERBOSE) dim.print();
|
||||
TKDNN_TSTART
|
||||
netRT->infer(dim, input_d);
|
||||
TKDNN_TSTOP
|
||||
if(TKDNN_VERBOSE) dim.print();
|
||||
stats.push_back(t_ns);
|
||||
}
|
||||
|
||||
{
|
||||
TKDNN_TSTART
|
||||
for(int bi=0; bi<cur_batches;++bi)
|
||||
postprocess(bi, apply_colormap);
|
||||
TKDNN_TSTOP
|
||||
stats_post.push_back(t_ns);
|
||||
}
|
||||
}
|
||||
|
||||
void updateOriginal(cv::Mat frame, bool apply_colormap=true){
|
||||
|
||||
std::vector<cv::Mat> splitted_frames;
|
||||
int H, W, net_H, net_W;
|
||||
int top = 0, bottom = 0, left = 0, right = 0;
|
||||
std::vector<std::pair<int,int>> pos;
|
||||
|
||||
{
|
||||
TKDNN_TSTART
|
||||
cv::Size original_size = frame.size();
|
||||
|
||||
frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0);
|
||||
H = frame.rows;
|
||||
W = frame.cols;
|
||||
net_H = netRT->input_dim.h;
|
||||
net_W = netRT->input_dim.w;
|
||||
|
||||
cv::Mat frame_cropped;
|
||||
|
||||
if( H <= net_H && W <= net_W ){ // smaller size wrt network
|
||||
top = (net_H - H)/2;
|
||||
bottom = net_H - H - top ;
|
||||
left = (net_W - W)/2;
|
||||
right = net_W - W - left ;
|
||||
cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) );
|
||||
splitted_frames.push_back(frame_cropped);
|
||||
}
|
||||
else{ //bigger size wrt network
|
||||
|
||||
|
||||
if(H < net_H || W < net_W){
|
||||
if(H < net_H){
|
||||
top = (net_H - H)/2;
|
||||
bottom = net_H - H - top ;
|
||||
}
|
||||
else{
|
||||
left = (net_W - W)/2;
|
||||
right = net_W - W - left ;
|
||||
}
|
||||
cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0));
|
||||
}
|
||||
|
||||
for(int x=0; x+net_W<=W ;){
|
||||
for(int y=0; y+net_H <=H ; ){
|
||||
cv::Rect roi(x, y, net_W, net_H);
|
||||
cv::Mat image_roi = frame(roi);
|
||||
splitted_frames.push_back(image_roi);
|
||||
pos.push_back(std::make_pair(x,y));
|
||||
|
||||
y += net_H;
|
||||
if(y == H)
|
||||
break;
|
||||
if(y + net_H > H) y = H - net_H;
|
||||
}
|
||||
x += net_W;
|
||||
if(x == W)
|
||||
break;
|
||||
if(x + net_W > W) x = W - net_W;
|
||||
}
|
||||
}
|
||||
|
||||
tk::dnn::dataDim_t idim = netRT->input_dim;
|
||||
|
||||
if(splitted_frames.size()> nBatches)
|
||||
FatalError(std::to_string(splitted_frames.size()) + " min batches required");
|
||||
|
||||
for(int bi=0; bi<splitted_frames.size();++bi){
|
||||
cv::split(splitted_frames[bi], bgr);
|
||||
for (int i = 0; i < idim.c; i++){
|
||||
int idx = i * splitted_frames[bi].rows * splitted_frames[bi].cols;
|
||||
int ch = idim.c-1 -i;
|
||||
memcpy((void *)&input[idx + idim.tot()*bi], (void *)bgr[ch].data, splitted_frames[bi].rows * splitted_frames[bi].cols * sizeof(dnnType));
|
||||
}
|
||||
|
||||
checkCuda(cudaMemcpyAsync(input_d+ idim.tot()*bi, input + idim.tot()*bi, idim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
|
||||
normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d);
|
||||
}
|
||||
TKDNN_TSTOP
|
||||
stats_pre.push_back(t_ns);
|
||||
}
|
||||
|
||||
tk::dnn::dataDim_t dim = netRT->input_dim;
|
||||
dim.n = splitted_frames.size();
|
||||
{
|
||||
if(TKDNN_VERBOSE) dim.print();
|
||||
TKDNN_TSTART
|
||||
netRT->infer(dim, input_d);
|
||||
TKDNN_TSTOP
|
||||
if(TKDNN_VERBOSE) dim.print();
|
||||
stats.push_back(t_ns);
|
||||
}
|
||||
|
||||
dataDim_t odim = netRT->output_dim;
|
||||
|
||||
std::vector<cv::Mat> out_img;
|
||||
|
||||
{
|
||||
TKDNN_TSTART
|
||||
|
||||
for(int bi=0; bi<splitted_frames.size();++bi){
|
||||
|
||||
dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
|
||||
|
||||
matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h);
|
||||
maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w);
|
||||
checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost));
|
||||
|
||||
dataDim_t vdim = odim;
|
||||
vdim.c = 1;
|
||||
|
||||
cv::Mat colored;
|
||||
|
||||
if(apply_colormap)
|
||||
colored = vizData2Mat(tmpOutData_h, vdim, netRT->input_dim.h, netRT->input_dim.w, 0, classes, classes);
|
||||
else{
|
||||
cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h);
|
||||
colored_fp32.convertTo(colored, CV_8UC1);
|
||||
}
|
||||
out_img.push_back(colored);
|
||||
}
|
||||
|
||||
|
||||
cv::Mat seg(frame.size(), out_img[0].type());
|
||||
if(out_img.size() == 1)
|
||||
{
|
||||
cv::Rect roi(left, top, W, H);
|
||||
seg = out_img[0](roi);
|
||||
}
|
||||
else{
|
||||
int bi=0;
|
||||
|
||||
if(top == 0 && left == 0){
|
||||
|
||||
for(int i=0; i<out_img.size(); ++i){
|
||||
cv::Mat roi_collage = seg(cv::Rect( pos[i].first ,pos[i].second,out_img[i].cols,out_img[i].rows));
|
||||
out_img[i].copyTo(roi_collage);
|
||||
}
|
||||
}
|
||||
else{
|
||||
FatalError("Not handled case")
|
||||
}
|
||||
}
|
||||
segmented[0] = seg;
|
||||
|
||||
TKDNN_TSTOP
|
||||
stats_post.push_back(t_ns);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to draw boundixg boxes and labels on a frame.
|
||||
*/
|
||||
cv::Mat draw(const int cur_batches=1) {
|
||||
for(int i=0; i<cur_batches; ++i){
|
||||
|
||||
cv::imshow("segmented", segmented[i]);
|
||||
cv::resizeWindow("segmented", cv::Size(512,288));
|
||||
cv::waitKey(1);
|
||||
}
|
||||
return segmented[0];
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}}
|
||||
|
||||
#endif /* SEGMENTATIONNN_H*/
|
||||
@@ -4,7 +4,7 @@
|
||||
#include "utils.h"
|
||||
|
||||
void activationELUForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
|
||||
void activationLEAKYForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
|
||||
void activationLEAKYForward(dnnType *srcData, dnnType *dstData, int size, float slope, cudaStream_t stream = cudaStream_t(0));
|
||||
void activationReLUCeilingForward(dnnType *srcData, dnnType *dstData, int size, const float ceiling, cudaStream_t stream = cudaStream_t(0));
|
||||
void activationLOGISTICForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
|
||||
void activationSIGMOIDForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
|
||||
@@ -24,7 +24,7 @@ void softmaxForward(float *input, int n, int batch, int batch_offset,
|
||||
int groups, int group_offset, int stride, float temp, float *output, cudaStream_t stream = cudaStream_t(0));
|
||||
|
||||
void shortcutForward(dnnType *srcData, dnnType *dstData, int n1, int c1, int h1, int w1, int s1,
|
||||
int n2, int c2, int h2, int w2, int s2,
|
||||
int n2, int c2, int h2, int w2, int s2, bool mul,
|
||||
cudaStream_t stream = cudaStream_t(0));
|
||||
|
||||
void upsampleForward(dnnType *srcData, dnnType *dstData,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#define KERNELSTHRUST_H
|
||||
|
||||
|
||||
#include <thrust/extrema.h>
|
||||
#include <thrust/sort.h>
|
||||
#include <thrust/execution_policy.h>
|
||||
#include <thrust/functional.h>
|
||||
@@ -9,6 +10,8 @@
|
||||
#include <thrust/iterator/constant_iterator.h>
|
||||
#include <thrust/gather.h>
|
||||
#include <thrust/copy.h>
|
||||
#include <thrust/device_ptr.h>
|
||||
|
||||
|
||||
#include "tkdnn.h"
|
||||
|
||||
@@ -36,4 +39,6 @@ void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_be
|
||||
void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin,
|
||||
dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out);
|
||||
|
||||
void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w);
|
||||
|
||||
#endif //KERNELSTHRUST_H
|
||||
@@ -4,9 +4,8 @@
|
||||
class ActivationLeakyRT : public IPlugin {
|
||||
|
||||
public:
|
||||
ActivationLeakyRT() {
|
||||
|
||||
|
||||
ActivationLeakyRT(float s) {
|
||||
slope = s;
|
||||
}
|
||||
|
||||
~ActivationLeakyRT(){
|
||||
@@ -42,13 +41,13 @@ public:
|
||||
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
|
||||
|
||||
activationLEAKYForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
|
||||
reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
|
||||
reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, slope, stream);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
virtual size_t getSerializationSize() override {
|
||||
return 1*sizeof(int);
|
||||
return 1*sizeof(int) + 1*sizeof(float);
|
||||
}
|
||||
|
||||
virtual void serialize(void* buffer) override {
|
||||
@@ -58,4 +57,5 @@ public:
|
||||
}
|
||||
|
||||
int size;
|
||||
float slope;
|
||||
};
|
||||
|
||||
@@ -4,10 +4,11 @@
|
||||
class ShortcutRT : public IPlugin {
|
||||
|
||||
public:
|
||||
ShortcutRT(tk::dnn::dataDim_t bdim) {
|
||||
ShortcutRT(tk::dnn::dataDim_t bdim, bool mul) {
|
||||
this->bc = bdim.c;
|
||||
this->bh = bdim.h;
|
||||
this->bw = bdim.w;
|
||||
this->mul = mul;
|
||||
}
|
||||
|
||||
~ShortcutRT(){
|
||||
@@ -47,15 +48,14 @@ public:
|
||||
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
|
||||
|
||||
checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
|
||||
for(int b=0; b < batchSize; ++b)
|
||||
shortcutForward(srcDataBack + b*bc*bh*bw, dstData + b*c*h*w, 1, c, h, w, 1, 1, bc, bh, bw, 1, stream);
|
||||
shortcutForward(srcDataBack, dstData, batchSize, c, h, w, 1, batchSize, bc, bh, bw, 1, mul, stream);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
virtual size_t getSerializationSize() override {
|
||||
return 6*sizeof(int);
|
||||
return 6*sizeof(int) + sizeof(bool);
|
||||
}
|
||||
|
||||
virtual void serialize(void* buffer) override {
|
||||
@@ -63,6 +63,7 @@ public:
|
||||
tk::dnn::writeBUF(buf, bc);
|
||||
tk::dnn::writeBUF(buf, bh);
|
||||
tk::dnn::writeBUF(buf, bw);
|
||||
tk::dnn::writeBUF(buf, mul);
|
||||
tk::dnn::writeBUF(buf, c);
|
||||
tk::dnn::writeBUF(buf, h);
|
||||
tk::dnn::writeBUF(buf, w);
|
||||
@@ -72,4 +73,5 @@ public:
|
||||
|
||||
int c, h, w;
|
||||
int bc, bh, bw;
|
||||
bool mul;
|
||||
};
|
||||
|
||||
@@ -40,7 +40,7 @@
|
||||
#define COL_PURPLEB "\033[1;35m"
|
||||
#define COL_CYANB "\033[1;36m"
|
||||
|
||||
#define TKDNN_VERBOSE 0
|
||||
#define TKDNN_VERBOSE 1
|
||||
|
||||
// Simple Timer
|
||||
#ifdef __linux__
|
||||
|
||||
@@ -72,6 +72,8 @@ do
|
||||
# ./test_imuodom &>> $out_file
|
||||
# print_output $? imuodom
|
||||
|
||||
test_net shelfnet
|
||||
test_net shelfnet_berkeley
|
||||
test_net yolo4
|
||||
test_net yolo4-csp
|
||||
test_net yolo4x
|
||||
|
||||
+5
-5
@@ -5,11 +5,12 @@
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
Activation::Activation(Network *net, int act_mode, const float ceiling) :
|
||||
Activation::Activation(Network *net, int act_mode, const float ceiling, const float slope) :
|
||||
Layer(net) {
|
||||
|
||||
this->act_mode = act_mode;
|
||||
this->ceiling = ceiling;
|
||||
this->act_mode = act_mode;
|
||||
this->ceiling = ceiling;
|
||||
this->slope = slope;
|
||||
checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
|
||||
|
||||
if(int(act_mode) < 100) {
|
||||
@@ -46,8 +47,7 @@ Activation::~Activation() {
|
||||
|
||||
dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) {
|
||||
if(act_mode == ACTIVATION_LEAKY) {
|
||||
activationLEAKYForward(srcData, dstData, dim.tot());
|
||||
|
||||
activationLEAKYForward(srcData, dstData, dim.tot(), this->slope);
|
||||
}
|
||||
else if(act_mode == ACTIVATION_MISH) {
|
||||
activationMishForward(srcData, dstData, dim.tot());
|
||||
|
||||
+2
-1
@@ -26,10 +26,11 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
|
||||
}
|
||||
|
||||
readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek);
|
||||
seek += outputs;
|
||||
|
||||
this->batchnorm = batchnorm;
|
||||
if(batchnorm) {
|
||||
seek += outputs;
|
||||
|
||||
readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek);
|
||||
seek += outputs;
|
||||
readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek);
|
||||
|
||||
+19
-7
@@ -237,6 +237,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) {
|
||||
return convert_layer(input, (Flatten*) l);
|
||||
if(type == LAYER_RESHAPE)
|
||||
return convert_layer(input, (Reshape*) l);
|
||||
if(type == LAYER_RESIZE)
|
||||
return convert_layer(input, (Resize*) l);
|
||||
if(type == LAYER_REORG)
|
||||
return convert_layer(input, (Reorg*) l);
|
||||
if(type == LAYER_REGION)
|
||||
@@ -390,13 +392,13 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) {
|
||||
|
||||
#if NV_TENSORRT_MAJOR < 6
|
||||
// plugin version
|
||||
IPlugin *plugin = new ActivationLeakyRT();
|
||||
IPlugin *plugin = new ActivationLeakyRT(l->slope);
|
||||
IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
|
||||
checkNULL(lRT);
|
||||
return lRT;
|
||||
#else
|
||||
IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kLEAKY_RELU);
|
||||
lRT->setAlpha(0.1);
|
||||
lRT->setAlpha(l->slope);
|
||||
checkNULL(lRT);
|
||||
return lRT;
|
||||
#endif
|
||||
@@ -479,13 +481,23 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Flatten *l) {
|
||||
ILayer* NetworkRT::convert_layer(ITensor *input, Reshape *l) {
|
||||
// std::cout<<"convert Reshape\n";
|
||||
|
||||
l->output_dim.print();
|
||||
IPlugin *plugin = new ReshapeRT(l->output_dim);
|
||||
IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
|
||||
checkNULL(lRT);
|
||||
return lRT;
|
||||
}
|
||||
|
||||
ILayer* NetworkRT::convert_layer(ITensor *input, Resize *l) {
|
||||
// std::cout<<"convert Resize\n";
|
||||
|
||||
IResizeLayer *lRT = networkRT->addResize(*input); //default is kNEAREST
|
||||
checkNULL(lRT);
|
||||
Dims d{};
|
||||
lRT->setResizeMode(ResizeMode(l->mode));
|
||||
lRT->setOutputDimensions(DimsCHW{l->output_dim.c, l->output_dim.h, l->output_dim.w});
|
||||
return lRT;
|
||||
}
|
||||
|
||||
ILayer* NetworkRT::convert_layer(ITensor *input, Reorg *l) {
|
||||
//std::cout<<"convert Reorg\n";
|
||||
|
||||
@@ -513,7 +525,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) {
|
||||
|
||||
ITensor *back_tens = tensors[l->backLayer];
|
||||
|
||||
if(l->backLayer->output_dim.c == l->output_dim.c)
|
||||
if(l->backLayer->output_dim.c == l->output_dim.c && !l->mul)
|
||||
{
|
||||
IElementWiseLayer *lRT = networkRT->addElementWise(*input, *back_tens, ElementWiseOperation::kSUM);
|
||||
checkNULL(lRT);
|
||||
@@ -522,7 +534,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) {
|
||||
else
|
||||
{
|
||||
// plugin version
|
||||
IPlugin *plugin = new ShortcutRT(l->backLayer->output_dim);
|
||||
IPlugin *plugin = new ShortcutRT(l->backLayer->output_dim, l->mul);
|
||||
ITensor **inputs = new ITensor*[2];
|
||||
inputs[0] = input;
|
||||
inputs[1] = back_tens;
|
||||
@@ -651,7 +663,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
|
||||
//std::cout<<name<<std::endl;
|
||||
|
||||
if(name.find("ActivationLeaky") == 0) {
|
||||
ActivationLeakyRT *a = new ActivationLeakyRT();
|
||||
ActivationLeakyRT *a = new ActivationLeakyRT(readBUF<float>(buf));
|
||||
a->size = readBUF<int>(buf);
|
||||
assert(buf == bufCheck + serialLength);
|
||||
return a;
|
||||
@@ -710,7 +722,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
|
||||
bdim.w = readBUF<int>(buf);
|
||||
bdim.l = 1;
|
||||
|
||||
ShortcutRT *r = new ShortcutRT(bdim);
|
||||
ShortcutRT *r = new ShortcutRT(bdim, readBUF<bool>(buf));
|
||||
r->c = readBUF<int>(buf);
|
||||
r->h = readBUF<int>(buf);
|
||||
r->w = readBUF<int>(buf);
|
||||
|
||||
+380
-15
@@ -6,23 +6,389 @@
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
cv::Mat vizFloat2colorMap(cv::Mat map) {
|
||||
cv::Mat mapillary_15_map(cv::Mat adjMap){
|
||||
|
||||
// cv::imshow("test", adjMap);
|
||||
// cv::waitKey(0);
|
||||
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
|
||||
|
||||
//animal
|
||||
M3.at<uchar>(0)=165;
|
||||
M2.at<uchar>(0)=42;
|
||||
M1.at<uchar>(0)=45;
|
||||
|
||||
//curb
|
||||
M3.at<uchar>(1)=196;
|
||||
M2.at<uchar>(1)=196;
|
||||
M1.at<uchar>(1)=196;
|
||||
|
||||
//barrier
|
||||
M3.at<uchar>(2)=90;
|
||||
M2.at<uchar>(2)=120;
|
||||
M1.at<uchar>(2)=150;
|
||||
|
||||
//road
|
||||
M3.at<uchar>(3)=128;
|
||||
M2.at<uchar>(3)=64;
|
||||
M1.at<uchar>(3)=128;
|
||||
|
||||
//building
|
||||
M3.at<uchar>(4)=70;
|
||||
M2.at<uchar>(4)=70;
|
||||
M1.at<uchar>(4)=70;
|
||||
|
||||
//person
|
||||
M3.at<uchar>(5)=220;
|
||||
M2.at<uchar>(5)=20;
|
||||
M1.at<uchar>(5)=60;
|
||||
|
||||
//roadmark
|
||||
M3.at<uchar>(6)=255;
|
||||
M2.at<uchar>(6)=255;
|
||||
M1.at<uchar>(6)=255;
|
||||
|
||||
//nature
|
||||
M3.at<uchar>(7)=107;
|
||||
M2.at<uchar>(7)=142;
|
||||
M1.at<uchar>(7)=35;
|
||||
|
||||
//sky
|
||||
M3.at<uchar>(8)=70;
|
||||
M2.at<uchar>(8)=130;
|
||||
M1.at<uchar>(8)=180;
|
||||
|
||||
//billboard
|
||||
M3.at<uchar>(9)=220;
|
||||
M2.at<uchar>(9)=220;
|
||||
M1.at<uchar>(9)=220;
|
||||
|
||||
//pole
|
||||
M3.at<uchar>(10)=153;
|
||||
M2.at<uchar>(10)=153;
|
||||
M1.at<uchar>(10)=153;
|
||||
|
||||
//traffic sign
|
||||
M3.at<uchar>(11)=128;
|
||||
M2.at<uchar>(11)=128;
|
||||
M1.at<uchar>(11)=128;
|
||||
|
||||
//bike
|
||||
M3.at<uchar>(12)=119;
|
||||
M2.at<uchar>(12)=11;
|
||||
M1.at<uchar>(12)=32;
|
||||
|
||||
//vehicle
|
||||
M3.at<uchar>(13)=0;
|
||||
M2.at<uchar>(13)=0;
|
||||
M1.at<uchar>(13)=142;
|
||||
|
||||
//void
|
||||
for(int i=14;i<256;i++)
|
||||
{
|
||||
M1.at<uchar>(i)=0;
|
||||
M2.at<uchar>(i)=0;
|
||||
M3.at<uchar>(i)=0;
|
||||
}
|
||||
|
||||
cv::Mat r1,r2,r3;
|
||||
|
||||
cv::LUT(adjMap,M1,r1);
|
||||
cv::LUT(adjMap,M2,r2);
|
||||
cv::LUT(adjMap,M3,r3);
|
||||
|
||||
std::vector<cv::Mat> planes;
|
||||
planes.push_back(r1);
|
||||
planes.push_back(r2);
|
||||
planes.push_back(r3);
|
||||
|
||||
cv::Mat dst;
|
||||
cv::merge(planes,dst);
|
||||
return dst;
|
||||
|
||||
|
||||
}
|
||||
|
||||
cv::Mat berkeley_20_map(cv::Mat adjMap){
|
||||
|
||||
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
|
||||
|
||||
//road
|
||||
M3.at<uchar>(0)=128;
|
||||
M2.at<uchar>(0)=64;
|
||||
M1.at<uchar>(0)=128;
|
||||
|
||||
//sidewalk
|
||||
M3.at<uchar>(1)=244;
|
||||
M2.at<uchar>(1)=35;
|
||||
M1.at<uchar>(1)=232;
|
||||
|
||||
//building
|
||||
M3.at<uchar>(2)=70;
|
||||
M2.at<uchar>(2)=70;
|
||||
M1.at<uchar>(2)=70;
|
||||
|
||||
//wall
|
||||
M3.at<uchar>(3)=102;
|
||||
M2.at<uchar>(3)=102;
|
||||
M1.at<uchar>(3)=156;
|
||||
|
||||
//fence
|
||||
M3.at<uchar>(4)=90;
|
||||
M2.at<uchar>(4)=120;
|
||||
M1.at<uchar>(4)=150;
|
||||
|
||||
//pole
|
||||
M3.at<uchar>(5)=153;
|
||||
M2.at<uchar>(5)=153;
|
||||
M1.at<uchar>(5)=153;
|
||||
|
||||
//traffic light
|
||||
M3.at<uchar>(6)=250;
|
||||
M2.at<uchar>(6)=170;
|
||||
M1.at<uchar>(6)=30;
|
||||
|
||||
//traffic sign
|
||||
M3.at<uchar>(7)=128;
|
||||
M2.at<uchar>(7)=128;
|
||||
M1.at<uchar>(7)=128;
|
||||
|
||||
//nature
|
||||
M3.at<uchar>(8)=107;
|
||||
M2.at<uchar>(8)=142;
|
||||
M1.at<uchar>(8)=35;
|
||||
|
||||
//ground
|
||||
M3.at<uchar>(9)=0;
|
||||
M2.at<uchar>(9)=192;
|
||||
M1.at<uchar>(9)=0;
|
||||
|
||||
//sky
|
||||
M3.at<uchar>(10)=70;
|
||||
M2.at<uchar>(10)=130;
|
||||
M1.at<uchar>(10)=180;
|
||||
|
||||
//person
|
||||
M3.at<uchar>(11)=220;
|
||||
M2.at<uchar>(11)=20;
|
||||
M1.at<uchar>(11)=60;
|
||||
|
||||
//rider
|
||||
M3.at<uchar>(12)=255;
|
||||
M2.at<uchar>(12)=0;
|
||||
M1.at<uchar>(12)=100;
|
||||
|
||||
//car
|
||||
M3.at<uchar>(13)=0;
|
||||
M2.at<uchar>(13)=0;
|
||||
M1.at<uchar>(13)=142;
|
||||
|
||||
//truck
|
||||
M3.at<uchar>(14)=0;
|
||||
M2.at<uchar>(14)=0;
|
||||
M1.at<uchar>(14)=70;
|
||||
|
||||
//bus
|
||||
M3.at<uchar>(15)=0;
|
||||
M2.at<uchar>(15)=60;
|
||||
M1.at<uchar>(15)=100;
|
||||
|
||||
//train
|
||||
M3.at<uchar>(16)=0;
|
||||
M2.at<uchar>(16)=0;
|
||||
M1.at<uchar>(16)=192;
|
||||
|
||||
//motorbike
|
||||
M3.at<uchar>(17)=0;
|
||||
M2.at<uchar>(17)=0;
|
||||
M1.at<uchar>(17)=230;
|
||||
|
||||
//bike
|
||||
M3.at<uchar>(18)=119;
|
||||
M2.at<uchar>(18)=11;
|
||||
M1.at<uchar>(18)=32;
|
||||
|
||||
//void
|
||||
for(int i=19;i<256;i++)
|
||||
{
|
||||
M1.at<uchar>(i)=0;
|
||||
M2.at<uchar>(i)=0;
|
||||
M3.at<uchar>(i)=0;
|
||||
}
|
||||
|
||||
cv::Mat r1,r2,r3;
|
||||
|
||||
cv::LUT(adjMap,M1,r1);
|
||||
cv::LUT(adjMap,M2,r2);
|
||||
cv::LUT(adjMap,M3,r3);
|
||||
|
||||
std::vector<cv::Mat> planes;
|
||||
planes.push_back(r1);
|
||||
planes.push_back(r2);
|
||||
planes.push_back(r3);
|
||||
|
||||
cv::Mat dst;
|
||||
cv::merge(planes,dst);
|
||||
return dst;
|
||||
|
||||
}
|
||||
|
||||
cv::Mat cityscapes_19_map(cv::Mat adjMap){
|
||||
|
||||
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
|
||||
|
||||
//road
|
||||
M3.at<uchar>(0)=128;
|
||||
M2.at<uchar>(0)=64;
|
||||
M1.at<uchar>(0)=128;
|
||||
|
||||
//sidewalk
|
||||
M3.at<uchar>(1)=244;
|
||||
M2.at<uchar>(1)=35;
|
||||
M1.at<uchar>(1)=232;
|
||||
|
||||
//building
|
||||
M3.at<uchar>(2)=70;
|
||||
M2.at<uchar>(2)=70;
|
||||
M1.at<uchar>(2)=70;
|
||||
|
||||
//wall
|
||||
M3.at<uchar>(3)=102;
|
||||
M2.at<uchar>(3)=102;
|
||||
M1.at<uchar>(3)=156;
|
||||
|
||||
//fence
|
||||
M3.at<uchar>(4)=190;
|
||||
M2.at<uchar>(4)=153;
|
||||
M1.at<uchar>(4)=153;
|
||||
|
||||
//pole
|
||||
M3.at<uchar>(5)=153;
|
||||
M2.at<uchar>(5)=153;
|
||||
M1.at<uchar>(5)=153;
|
||||
|
||||
//traffic light
|
||||
M3.at<uchar>(6)=250;
|
||||
M2.at<uchar>(6)=170;
|
||||
M1.at<uchar>(6)=30;
|
||||
|
||||
//traffic sign
|
||||
M3.at<uchar>(7)=220;
|
||||
M2.at<uchar>(7)=220;
|
||||
M1.at<uchar>(7)=0;
|
||||
|
||||
//vegetation
|
||||
M3.at<uchar>(8)=107;
|
||||
M2.at<uchar>(8)=142;
|
||||
M1.at<uchar>(8)=35;
|
||||
|
||||
//terrain
|
||||
M3.at<uchar>(9)=152;
|
||||
M2.at<uchar>(9)=251;
|
||||
M1.at<uchar>(9)=152;
|
||||
|
||||
//sky
|
||||
M3.at<uchar>(10)=70;
|
||||
M2.at<uchar>(10)=130;
|
||||
M1.at<uchar>(10)=180;
|
||||
|
||||
//person
|
||||
M3.at<uchar>(11)=220;
|
||||
M2.at<uchar>(11)=20;
|
||||
M1.at<uchar>(11)=60;
|
||||
|
||||
//rider
|
||||
M3.at<uchar>(12)=255;
|
||||
M2.at<uchar>(12)=0;
|
||||
M1.at<uchar>(12)=0;
|
||||
|
||||
//car
|
||||
M3.at<uchar>(13)=0;
|
||||
M2.at<uchar>(13)=0;
|
||||
M1.at<uchar>(13)=142;
|
||||
|
||||
//truck
|
||||
M3.at<uchar>(14)=0;
|
||||
M2.at<uchar>(14)=0;
|
||||
M1.at<uchar>(14)=70;
|
||||
|
||||
//bus
|
||||
M3.at<uchar>(15)=0;
|
||||
M2.at<uchar>(15)=60;
|
||||
M1.at<uchar>(15)=100;
|
||||
|
||||
//train
|
||||
M3.at<uchar>(16)=0;
|
||||
M2.at<uchar>(16)=80;
|
||||
M1.at<uchar>(16)=100;
|
||||
|
||||
//motorcycle
|
||||
M3.at<uchar>(17)=0;
|
||||
M2.at<uchar>(17)=0;
|
||||
M1.at<uchar>(17)=230;
|
||||
|
||||
//bicycle
|
||||
M3.at<uchar>(18)=119;
|
||||
M2.at<uchar>(18)=11;
|
||||
M1.at<uchar>(18)=32;
|
||||
|
||||
//void
|
||||
for(int i=19;i<256;i++)
|
||||
{
|
||||
M1.at<uchar>(i)=0;
|
||||
M2.at<uchar>(i)=0;
|
||||
M3.at<uchar>(i)=0;
|
||||
}
|
||||
|
||||
cv::Mat r1,r2,r3;
|
||||
|
||||
cv::LUT(adjMap,M1,r1);
|
||||
cv::LUT(adjMap,M2,r2);
|
||||
cv::LUT(adjMap,M3,r3);
|
||||
|
||||
std::vector<cv::Mat> planes;
|
||||
planes.push_back(r1);
|
||||
planes.push_back(r2);
|
||||
planes.push_back(r3);
|
||||
|
||||
cv::Mat dst;
|
||||
cv::merge(planes,dst);
|
||||
return dst;
|
||||
|
||||
}
|
||||
|
||||
|
||||
cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, int classes) {
|
||||
|
||||
if(min == 0 && max == 0)
|
||||
cv::minMaxIdx(map, &min, &max);
|
||||
|
||||
double min;
|
||||
double max;
|
||||
cv::minMaxIdx(map, &min, &max);
|
||||
cv::Mat adjMap;
|
||||
// expand your range to 0..255. Similar to histEq();
|
||||
map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min);
|
||||
//return adjMap;
|
||||
|
||||
|
||||
cv::Mat falseColorsMap;
|
||||
applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_HOT);
|
||||
|
||||
switch (classes)
|
||||
{
|
||||
case 15:
|
||||
map.convertTo(adjMap,CV_8UC1);
|
||||
falseColorsMap = mapillary_15_map(adjMap);
|
||||
break;
|
||||
case 20:
|
||||
map.convertTo(adjMap,CV_8UC1);
|
||||
falseColorsMap = berkeley_20_map(adjMap);
|
||||
break;
|
||||
case 19:
|
||||
map.convertTo(adjMap,CV_8UC1);
|
||||
falseColorsMap = cityscapes_19_map(adjMap);
|
||||
break;
|
||||
|
||||
default:
|
||||
// expand your range to 0..255. Similar to histEq();
|
||||
map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min);
|
||||
applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_JET);
|
||||
}
|
||||
return falseColorsMap;
|
||||
}
|
||||
|
||||
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) {
|
||||
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min, double max, int classes) {
|
||||
dnnType *data = nullptr;
|
||||
|
||||
// copy to CPU
|
||||
@@ -38,14 +404,13 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) {
|
||||
cv::Mat grid = cv::Mat(gridSize, CV_8UC3, cv::Scalar(0));
|
||||
|
||||
for(int i=0; i<dim.c;i++) {
|
||||
cv::Mat raw = vizFloat2colorMap(cv::Mat(cv::Size(dim.w, dim.h),CV_32FC1, data + dim.w*dim.h*i));
|
||||
cv::Mat raw = vizFloat2colorMap(cv::Mat(cv::Size(dim.w, dim.h),CV_32FC1, data + dim.w*dim.h*i), min, max, classes);
|
||||
int r = i / gridDim;
|
||||
int c = i - r * gridDim;
|
||||
raw.copyTo(grid.rowRange(r*dim.h, r*dim.h + dim.h).colRange(c*dim.w, c*dim.w + dim.w));
|
||||
}
|
||||
|
||||
float ar = float(dim.w)/dim.h;
|
||||
cv::Size vdim(ar*imgdim, imgdim);
|
||||
cv::Size vdim(img_w, img_h);
|
||||
cv::Mat viz;
|
||||
cv::resize(grid, viz, vdim, 0, 0, 0);
|
||||
|
||||
@@ -59,7 +424,7 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) {
|
||||
cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim) {
|
||||
if(layer >= net->num_layers)
|
||||
FatalError("Could not viz layer\n");
|
||||
return vizData2Mat(net->layers[layer]->dstData, net->layers[layer]->output_dim, imgdim);
|
||||
return vizData2Mat(net->layers[layer]->dstData, net->layers[layer]->output_dim, imgdim, imgdim);
|
||||
|
||||
//cv::imwrite("viz/layer" + std::to_string(layer) + ".png", viz);
|
||||
//cv::imshow("layer", viz);
|
||||
|
||||
@@ -15,6 +15,11 @@ Reshape::Reshape(Network *net, dataDim_t new_dim) : Layer(net) {
|
||||
output_dim.w = new_dim.w;
|
||||
output_dim.l = new_dim.l;
|
||||
|
||||
output_dim = new_dim;
|
||||
|
||||
if(input_dim.tot() != output_dim.tot())
|
||||
FatalError("Reshape dimension mismatch");
|
||||
|
||||
}
|
||||
|
||||
Reshape::~Reshape() {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
#include <iostream>
|
||||
|
||||
#include "Layer.h"
|
||||
#include "kernels.h"
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
Resize::Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed, ResizeMode_t mode) : Layer(net) {
|
||||
|
||||
this->mode = mode;
|
||||
if(fixed){
|
||||
output_dim.c = scale_c;
|
||||
output_dim.h = scale_h;
|
||||
output_dim.w = scale_w;
|
||||
}
|
||||
else{
|
||||
output_dim.c *= scale_c;
|
||||
output_dim.h *= scale_h;
|
||||
output_dim.w *= scale_w;
|
||||
}
|
||||
|
||||
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
|
||||
}
|
||||
|
||||
Resize::~Resize() {
|
||||
|
||||
checkCuda( cudaFree(dstData) );
|
||||
}
|
||||
|
||||
dnnType* Resize::infer(dataDim_t &dim, dnnType* srcData) {
|
||||
|
||||
resizeForward(srcData, dstData, dim.n, dim.c, dim.h, dim.w,
|
||||
output_dim.c, output_dim.h, output_dim.w);
|
||||
dim = output_dim;
|
||||
|
||||
return dstData;
|
||||
}
|
||||
|
||||
}}
|
||||
+7
-6
@@ -5,15 +5,16 @@
|
||||
|
||||
namespace tk { namespace dnn {
|
||||
|
||||
Shortcut::Shortcut(Network *net, Layer *backLayer) : Layer(net) {
|
||||
Shortcut::Shortcut(Network *net, Layer *backLayer, bool mul) : Layer(net) {
|
||||
|
||||
this->backLayer = backLayer;
|
||||
this->mul = mul;
|
||||
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
|
||||
|
||||
if( /*backLayer->output_dim.c != input_dim.c ||*/
|
||||
backLayer->output_dim.w != input_dim.w ||
|
||||
backLayer->output_dim.h != input_dim.h )
|
||||
FatalError("Shortcut dim mismatch");
|
||||
if( ( backLayer->output_dim.c != input_dim.c && mul ) ||
|
||||
(( backLayer->output_dim.w != input_dim.w || backLayer->output_dim.h != input_dim.h ) && !mul ) )
|
||||
FatalError("Shortcut dim missmatch");
|
||||
|
||||
}
|
||||
|
||||
Shortcut::~Shortcut() {
|
||||
@@ -26,7 +27,7 @@ dnnType* Shortcut::infer(dataDim_t &dim, dnnType* srcData) {
|
||||
dataDim_t bdim = this->backLayer->output_dim;
|
||||
|
||||
checkCuda(cudaMemcpy(dstData, srcData, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
|
||||
shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1);
|
||||
shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1, mul);
|
||||
|
||||
//update data dimensions
|
||||
dim = output_dim;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#include "kernels.h"
|
||||
|
||||
__global__
|
||||
void activation_leaky(dnnType *input, dnnType *output, int size) {
|
||||
void activation_leaky(dnnType *input, dnnType *output, int size, float slope) {
|
||||
|
||||
int i = blockDim.x*blockIdx.x + threadIdx.x;
|
||||
|
||||
@@ -9,7 +9,7 @@ void activation_leaky(dnnType *input, dnnType *output, int size) {
|
||||
if (input[i]>0)
|
||||
output[i] = input[i];
|
||||
else
|
||||
output[i] = 0.1f*input[i];
|
||||
output[i] = slope*input[i];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,12 +17,12 @@ void activation_leaky(dnnType *input, dnnType *output, int size) {
|
||||
/**
|
||||
ELU activation function
|
||||
*/
|
||||
void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream)
|
||||
void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, float slope, cudaStream_t stream)
|
||||
{
|
||||
int blocks = (size+255)/256;
|
||||
int threads = 256;
|
||||
|
||||
activation_leaky<<<blocks, threads, 0, stream>>>(srcData, dstData, size);
|
||||
activation_leaky<<<blocks, threads, 0, stream>>>(srcData, dstData, size, slope);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -34,6 +34,25 @@ void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int
|
||||
sortAndTopK_kernel<<<blocks, threads, 0>>>(src_begin, idsrc, topk_scores, topk_inds, topk_ys, topk_xs, size, K);
|
||||
}
|
||||
|
||||
__global__
|
||||
void maxElem_kernel(float *src_begin, float *dst_begin, const int n_classes, const int size){
|
||||
int i = blockDim.x*blockIdx.x + threadIdx.x;
|
||||
if (i > size)
|
||||
return;
|
||||
|
||||
thrust::device_ptr<float> dPbeg ( &src_begin[i*n_classes] ) ;
|
||||
thrust::device_ptr<float> dPend = dPbeg + n_classes;
|
||||
thrust::device_ptr<float> result = thrust::max_element(thrust::device,dPbeg, dPend);
|
||||
|
||||
dst_begin[i] = result - dPbeg;
|
||||
}
|
||||
|
||||
void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w){
|
||||
int blocks = (h*w)/32+1;
|
||||
int threads = 32;
|
||||
maxElem_kernel<<<blocks, threads, 0>>>(src_begin, dst_begin, c, h*w);
|
||||
}
|
||||
|
||||
void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys){
|
||||
thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), clses, thrust::divides<int>());
|
||||
thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), ids_begin, thrust::modulus<int>());
|
||||
|
||||
+14
-27
@@ -1,46 +1,33 @@
|
||||
#include "kernels.h"
|
||||
#include <stdio.h>
|
||||
#define MIN(a,b) (((a)<(b))?(a):(b))
|
||||
#define MAX(a,b) (((a)>(b))?(a):(b))
|
||||
|
||||
__global__ void resize_kernel( int i_N,float *x, int i_w, int i_h, int i_c,
|
||||
__global__ void resize_kernel( int size,float *x, int i_w, int i_h, int i_c,
|
||||
int o_w, int o_h, int o_c, int batch, float *out)
|
||||
{
|
||||
int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
|
||||
if(i >= i_N) return;
|
||||
int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
|
||||
if(id >= size) return;
|
||||
|
||||
int out_index = i;
|
||||
int out_w = i%o_w;
|
||||
i = i/o_w;
|
||||
int out_h = i%o_h;
|
||||
i = i/o_h;
|
||||
int out_c = i%o_c;
|
||||
i = i/o_c;
|
||||
int i = id % o_w;
|
||||
id /= o_w;
|
||||
int j = id % o_h;
|
||||
id /= o_h;
|
||||
int k = id % o_c;
|
||||
id /= o_c;
|
||||
int b = id % batch;
|
||||
|
||||
//copying last column/last row as padding
|
||||
int in_index = ((i*i_c + MIN(out_c,i_c-1))*i_h + MIN(out_h,i_h-1))*i_w + MIN(out_w, i_w-1);
|
||||
out[out_index] = x[in_index];
|
||||
int out_index = i + o_w*(j + o_h*(k + o_c*b));
|
||||
int add_index = i/(o_w/i_w) + i_w*(j/(o_h/i_h) + i_h*(k + i_c*b));
|
||||
out[out_index] = x[add_index];
|
||||
}
|
||||
|
||||
|
||||
void resizeForward( dnnType* srcData, dnnType* dstData, int n, int i_c, int i_h, int i_w,
|
||||
int o_c, int o_h, int o_w, cudaStream_t stream )
|
||||
{
|
||||
int i_size = n*i_c*i_h*i_w;
|
||||
int o_size = n*o_c*o_h*o_w;
|
||||
|
||||
int blocks = (o_size+255)/256;
|
||||
int threads = 256;
|
||||
|
||||
if(i_c == o_c && i_h == o_h && i_w == o_w )
|
||||
{
|
||||
checkCuda(cudaMemcpy(dstData, srcData, i_size*sizeof(dnnType), cudaMemcpyDeviceToDevice));
|
||||
}
|
||||
else
|
||||
{
|
||||
checkCuda(cudaMemset(dstData, 0, o_size*sizeof(dnnType)));
|
||||
resize_kernel<<<blocks, threads, 0, stream>>>(o_size, srcData, i_w, i_h, i_c, o_w, o_h, o_c, n, dstData);
|
||||
// printDeviceVector(i_size, srcData);
|
||||
// printDeviceVector(o_size, dstData);
|
||||
}
|
||||
resize_kernel<<<blocks, threads, 0, stream>>>(o_size, srcData, i_w, i_h, i_c, o_w, o_h, o_c, n, dstData);
|
||||
}
|
||||
|
||||
+48
-15
@@ -21,27 +21,60 @@ __global__ void shortcut_kernel(int size, int minw, int minh, int minc, int stri
|
||||
//out[out_index] += add[add_index];
|
||||
}
|
||||
|
||||
__global__ void shortcut_mul_kernel(int size, int minw, int minh, int minc, int sample, int batch,
|
||||
int w1, int h1, int c1, dnnType *mul,
|
||||
int w2, int h2, int c2, float s1, float s2, dnnType *out)
|
||||
{
|
||||
int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
|
||||
if (id >= size) return;
|
||||
int i = id % minw;
|
||||
id /= minw;
|
||||
int j = id % minh;
|
||||
id /= minh;
|
||||
int k = id % minc;
|
||||
id /= minc;
|
||||
int b = id % batch;
|
||||
|
||||
int out_index = i*sample + w1*(j*sample + h1*(k + c1*b));
|
||||
out[out_index] = out[out_index] * mul[k + c2*b];
|
||||
}
|
||||
|
||||
void shortcutForward(dnnType* srcData, dnnType* dstData, int n1, int c1, int h1, int w1, int s1,
|
||||
int n2, int c2, int h2, int w2, int s2,
|
||||
cudaStream_t stream)
|
||||
bool mul, cudaStream_t stream)
|
||||
{
|
||||
assert(n1 == n2);
|
||||
int batch = n1;
|
||||
|
||||
int minw = (w1 < w2) ? w1 : w2;
|
||||
int minh = (h1 < h2) ? h1 : h2;
|
||||
int minc = (c1 < c2) ? c1 : c2;
|
||||
if(!mul){
|
||||
int minw = (w1 < w2) ? w1 : w2;
|
||||
int minh = (h1 < h2) ? h1 : h2;
|
||||
int minc = (c1 < c2) ? c1 : c2;
|
||||
int stride = w1/w2;
|
||||
int sample = w2/w1;
|
||||
assert(stride == h1/h2);
|
||||
assert(sample == h2/h1);
|
||||
if(stride < 1) stride = 1;
|
||||
if(sample < 1) sample = 1;
|
||||
|
||||
int stride = w1/w2;
|
||||
int sample = w2/w1;
|
||||
assert(stride == h1/h2);
|
||||
assert(sample == h2/h1);
|
||||
if(stride < 1) stride = 1;
|
||||
if(sample < 1) sample = 1;
|
||||
int size = batch * minw * minh * minc;
|
||||
int blocks = (size+255)/256;
|
||||
int threads = 256;
|
||||
|
||||
shortcut_kernel<<<blocks, threads, 0, stream>>>(size, minw, minh, minc, stride, sample, batch,
|
||||
w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData);
|
||||
}
|
||||
else{
|
||||
int minw = w1;
|
||||
int minh = h1;
|
||||
int minc = c1;
|
||||
int sample = 1;
|
||||
|
||||
int size = batch * minw * minh * minc;
|
||||
int blocks = (size+255)/256;
|
||||
int threads = 256;
|
||||
shortcut_kernel<<<blocks, threads, 0, stream>>>(size, minw, minh, minc, stride, sample, batch,
|
||||
w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData);
|
||||
int size = batch * minw * minh * minc;
|
||||
int blocks = (size+255)/256;
|
||||
int threads = 256;
|
||||
|
||||
shortcut_mul_kernel<<<blocks, threads, 0, stream>>>(size, minw, minh, minc, sample, batch,
|
||||
w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,6 +111,7 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int
|
||||
}
|
||||
int diffs = 0;
|
||||
for(int i=0; i<size; i++) {
|
||||
// data_h[i] = data_h[i]*1e-2;
|
||||
if(data_h[i] != data_h[i] || correct_h[i] != correct_h[i] || //nan control
|
||||
fabs(data_h[i] - correct_h[i]) > eps) {
|
||||
diffs += 1;
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
#include <iostream>
|
||||
#include <opencv2/highgui/highgui.hpp>
|
||||
#include <opencv2/imgproc/imgproc.hpp>
|
||||
|
||||
#include "tkdnn.h"
|
||||
#include "NetworkViz.h"
|
||||
|
||||
|
||||
const char *input_bin = "shelfnet/debug/input.bin";
|
||||
|
||||
const char *backbone[] = {
|
||||
"shelfnet/layers/backbone-conv1.bin",
|
||||
"shelfnet/layers/backbone-layer1-0-conv1.bin",
|
||||
"shelfnet/layers/backbone-layer1-0-conv2.bin",
|
||||
"shelfnet/layers/backbone-layer1-1-conv1.bin",
|
||||
"shelfnet/layers/backbone-layer1-1-conv2.bin",
|
||||
"shelfnet/layers/backbone-layer2-0-conv1.bin",
|
||||
"shelfnet/layers/backbone-layer2-0-conv2.bin",
|
||||
"shelfnet/layers/backbone-layer2-0-downsample-0.bin",
|
||||
"shelfnet/layers/backbone-layer2-1-conv1.bin",
|
||||
"shelfnet/layers/backbone-layer2-1-conv2.bin",
|
||||
"shelfnet/layers/backbone-layer3-0-conv1.bin",
|
||||
"shelfnet/layers/backbone-layer3-0-conv2.bin",
|
||||
"shelfnet/layers/backbone-layer3-0-downsample-0.bin",
|
||||
"shelfnet/layers/backbone-layer3-1-conv1.bin",
|
||||
"shelfnet/layers/backbone-layer3-1-conv2.bin",
|
||||
"shelfnet/layers/backbone-layer4-0-conv1.bin",
|
||||
"shelfnet/layers/backbone-layer4-0-conv2.bin",
|
||||
"shelfnet/layers/backbone-layer4-0-downsample-0.bin",
|
||||
"shelfnet/layers/backbone-layer4-1-conv1.bin",
|
||||
"shelfnet/layers/backbone-layer4-1-conv2.bin"};
|
||||
|
||||
const char *conv_out[] = {
|
||||
"shelfnet/layers/conv_out-conv-conv.bin",
|
||||
"shelfnet/layers/conv_out-conv_out.bin",
|
||||
"shelfnet/layers/conv_out16-conv-conv.bin",
|
||||
"shelfnet/layers/conv_out16-conv_out.bin",
|
||||
"shelfnet/layers/conv_out32-conv-conv.bin",
|
||||
"shelfnet/layers/conv_out32-conv_out.bin"
|
||||
};
|
||||
|
||||
const char *decoder[] = {
|
||||
"shelfnet/layers/decoder-bottom-conv1.bin",
|
||||
"shelfnet/layers/decoder-bottom-conv12.bin",
|
||||
"shelfnet/layers/decoder-up_conv_list-0-conv-conv.bin",
|
||||
"shelfnet/layers/decoder-up_conv_list-0-conv_atten.bin",
|
||||
"shelfnet/layers/decoder-up_dense_list-0-conv.bin",
|
||||
"shelfnet/layers/decoder-up_conv_list-1-conv-conv.bin",
|
||||
"shelfnet/layers/decoder-up_conv_list-1-conv_atten.bin",
|
||||
"shelfnet/layers/decoder-up_dense_list-1-conv.bin"
|
||||
};
|
||||
|
||||
|
||||
const char *ladder[] = {
|
||||
"shelfnet/layers/ladder-inconv-conv1.bin",
|
||||
"shelfnet/layers/ladder-inconv-conv12.bin",
|
||||
"shelfnet/layers/ladder-down_module_list-0-conv1.bin",
|
||||
"shelfnet/layers/ladder-down_module_list-0-conv12.bin",
|
||||
"shelfnet/layers/ladder-down_conv_list-0.bin",
|
||||
|
||||
"shelfnet/layers/ladder-down_module_list-1-conv1.bin",
|
||||
"shelfnet/layers/ladder-down_module_list-1-conv12.bin",
|
||||
"shelfnet/layers/ladder-down_conv_list-1.bin",
|
||||
|
||||
"shelfnet/layers/ladder-bottom-conv1.bin",
|
||||
"shelfnet/layers/ladder-bottom-conv12.bin",
|
||||
|
||||
|
||||
|
||||
"shelfnet/layers/ladder-up_conv_list-0-conv-conv.bin",
|
||||
"shelfnet/layers/ladder-up_conv_list-0-conv_atten.bin",
|
||||
"shelfnet/layers/ladder-up_dense_list-0-conv.bin",
|
||||
|
||||
|
||||
"shelfnet/layers/ladder-up_conv_list-1-conv-conv.bin",
|
||||
"shelfnet/layers/ladder-up_conv_list-1-conv_atten.bin",
|
||||
"shelfnet/layers/ladder-up_dense_list-1-conv.bin"};
|
||||
|
||||
const char *trans[] = {
|
||||
"shelfnet/layers/trans1-conv.bin",
|
||||
"shelfnet/layers/trans2-conv.bin",
|
||||
"shelfnet/layers/trans3-conv.bin"};
|
||||
int main()
|
||||
{
|
||||
|
||||
downloadWeightsifDoNotExist(input_bin, "shelfnet", "https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download");
|
||||
|
||||
int classes = 19;
|
||||
|
||||
// Network layout
|
||||
tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1);
|
||||
tk::dnn::Network net(dim);
|
||||
|
||||
int bi = 0, di = 0, li = 0, ci = 0;
|
||||
new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX);
|
||||
|
||||
|
||||
|
||||
for(int i=0; i<2; ++i){
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
}
|
||||
|
||||
std::vector<tk::dnn::Layer*> features;
|
||||
for(int i=0;i<3;++i){
|
||||
int out_channel = pow(2,7+i);
|
||||
std::cout<<out_channel<<std::endl;
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 2, 2, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
tk::dnn::Layer* bn2 = new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Route(&net, &last, 1);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 2, 2, 0, 0, backbone[bi++], true);
|
||||
new tk::dnn::Shortcut(&net, bn2);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
features.push_back(last);
|
||||
}
|
||||
|
||||
for(int i=0; i<features.size(); ++i){
|
||||
new tk::dnn::Route(&net, &features[i], 1);
|
||||
int out_channel = pow(2,6+i);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, trans[i], true);
|
||||
features[i] = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
}
|
||||
|
||||
//DECODER
|
||||
|
||||
last = features[2];
|
||||
std::vector<tk::dnn::Layer*> up_out;
|
||||
//bottom
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
up_out.push_back(last);
|
||||
|
||||
for(int i=0; i<2; ++i){
|
||||
int out_channel = pow(2,7-i);
|
||||
//up-conv
|
||||
std::cout<<out_channel<<std::endl;
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
|
||||
new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true);
|
||||
|
||||
tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID);
|
||||
new tk::dnn::Route(&net, &last, 1);
|
||||
new tk::dnn::Shortcut(&net, act, true);
|
||||
|
||||
//interpolate
|
||||
new tk::dnn::Resize(&net, 1,2,2);
|
||||
new tk::dnn::Shortcut(&net, features[1-i]);
|
||||
|
||||
//up-dense
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
up_out.push_back(last);
|
||||
}
|
||||
|
||||
//LADDER
|
||||
|
||||
std::vector<tk::dnn::Layer*> down_out;
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
for(int i=0; i<2;++i){
|
||||
int out_channel = pow(2,6+i);
|
||||
tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]);
|
||||
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, l_last);
|
||||
l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
down_out.push_back(l_last);
|
||||
|
||||
new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU
|
||||
}
|
||||
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
up_out.clear();
|
||||
up_out.push_back(last);
|
||||
|
||||
for(int i=0; i<2; ++i){
|
||||
int out_channel = pow(2,7-i);
|
||||
//up-conv
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
|
||||
new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true);
|
||||
|
||||
tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID);
|
||||
new tk::dnn::Route(&net, &last, 1);
|
||||
new tk::dnn::Shortcut(&net, act, true);
|
||||
|
||||
//interpolate
|
||||
new tk::dnn::Resize(&net, 1,2,2);
|
||||
new tk::dnn::Shortcut(&net, down_out[1-i]);
|
||||
|
||||
// //up-dense
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
up_out.push_back(last);
|
||||
}
|
||||
|
||||
|
||||
// for(int i=2;i>=0;--i){
|
||||
// new tk::dnn::Route(&net, &up_out[i], 1);
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 19, 3, 3, 1, 1, 1, 1, conv_out[ci++], false);
|
||||
/*up_out[i] =*/ new tk::dnn::Resize(&net, 19, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR);
|
||||
// }
|
||||
|
||||
new tk::dnn::Softmax(&net);
|
||||
|
||||
const char *output_bin = "shelfnet/debug/softmax.bin";
|
||||
|
||||
// Load input
|
||||
dnnType *data;
|
||||
dnnType *input_h;
|
||||
readBinaryFile(input_bin, dim.tot(), &input_h, &data);
|
||||
std::cout<<"Input:"<<std::endl;
|
||||
|
||||
//print network model
|
||||
net.print();
|
||||
|
||||
// // convert network to tensorRT
|
||||
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("shelfnet"));
|
||||
|
||||
tk::dnn::dataDim_t dim1 = dim; //input dim
|
||||
dnnType *cudnn_out = nullptr;
|
||||
printCenteredTitle(" CUDNN inference ", '=', 30);
|
||||
{
|
||||
dim1.print();
|
||||
TKDNN_TSTART
|
||||
cudnn_out = net.infer(dim1, data);
|
||||
TKDNN_TSTOP
|
||||
dim1.print();
|
||||
}
|
||||
|
||||
tk::dnn::dataDim_t dim2 = dim;
|
||||
printCenteredTitle(" TENSORRT inference ", '=', 30);
|
||||
{
|
||||
dim2.print();
|
||||
TKDNN_TSTART
|
||||
netRT.infer(dim2, data);
|
||||
TKDNN_TSTOP
|
||||
dim2.print();
|
||||
}
|
||||
|
||||
dnnType *rt_out1 = (dnnType *)netRT.buffersRT[1];
|
||||
|
||||
printCenteredTitle(std::string(" CHECK RESULTS ").c_str(), '=', 30);
|
||||
dnnType *out1, *out1_h;
|
||||
int odim1 = dim1.tot();
|
||||
readBinaryFile(output_bin, odim1, &out1_h, &out1);
|
||||
|
||||
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
|
||||
std::cout << "CUDNN vs correct" << std::endl;
|
||||
ret_cudnn |= checkResult(odim1, cudnn_out, out1, true, 20) == 0 ? 0 : ERROR_CUDNN;
|
||||
|
||||
std::cout << "TRT vs correct" << std::endl;
|
||||
ret_tensorrt |=checkResult(odim1, rt_out1, out1) == 0 ? 0 : ERROR_TENSORRT;
|
||||
|
||||
std::cout << "CUDNN vs TRT " << std::endl;
|
||||
ret_cudnn_tensorrt |= checkResult(odim1, cudnn_out, rt_out1) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
|
||||
|
||||
cv::Mat viz = vizLayer2Mat(&net, net.num_layers-1);
|
||||
cv::imwrite("test.png", viz);
|
||||
|
||||
return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
#include <iostream>
|
||||
#include <opencv2/highgui/highgui.hpp>
|
||||
#include <opencv2/imgproc/imgproc.hpp>
|
||||
|
||||
#include "tkdnn.h"
|
||||
#include "NetworkViz.h"
|
||||
|
||||
|
||||
const char *input_bin = "shelfnet_berkeley/debug/input.bin";
|
||||
|
||||
const char *backbone[] = {
|
||||
"shelfnet_berkeley/layers/backbone-conv1.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer1-0-conv1.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer1-0-conv2.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer1-1-conv1.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer1-1-conv2.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer2-0-conv1.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer2-0-conv2.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer2-0-downsample-0.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer2-1-conv1.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer2-1-conv2.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer3-0-conv1.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer3-0-conv2.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer3-0-downsample-0.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer3-1-conv1.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer3-1-conv2.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer4-0-conv1.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer4-0-conv2.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer4-0-downsample-0.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer4-1-conv1.bin",
|
||||
"shelfnet_berkeley/layers/backbone-layer4-1-conv2.bin"};
|
||||
|
||||
const char *conv_out[] = {
|
||||
"shelfnet_berkeley/layers/conv_out-conv-conv.bin",
|
||||
"shelfnet_berkeley/layers/conv_out-conv_out.bin",
|
||||
"shelfnet_berkeley/layers/conv_out16-conv-conv.bin",
|
||||
"shelfnet_berkeley/layers/conv_out16-conv_out.bin",
|
||||
"shelfnet_berkeley/layers/conv_out32-conv-conv.bin",
|
||||
"shelfnet_berkeley/layers/conv_out32-conv_out.bin"
|
||||
};
|
||||
|
||||
const char *decoder[] = {
|
||||
"shelfnet_berkeley/layers/decoder-bottom-conv1.bin",
|
||||
"shelfnet_berkeley/layers/decoder-bottom-conv12.bin",
|
||||
"shelfnet_berkeley/layers/decoder-up_conv_list-0-conv-conv.bin",
|
||||
"shelfnet_berkeley/layers/decoder-up_conv_list-0-conv_atten.bin",
|
||||
"shelfnet_berkeley/layers/decoder-up_dense_list-0-conv.bin",
|
||||
"shelfnet_berkeley/layers/decoder-up_conv_list-1-conv-conv.bin",
|
||||
"shelfnet_berkeley/layers/decoder-up_conv_list-1-conv_atten.bin",
|
||||
"shelfnet_berkeley/layers/decoder-up_dense_list-1-conv.bin"
|
||||
};
|
||||
|
||||
|
||||
const char *ladder[] = {
|
||||
"shelfnet_berkeley/layers/ladder-inconv-conv1.bin",
|
||||
"shelfnet_berkeley/layers/ladder-inconv-conv12.bin",
|
||||
"shelfnet_berkeley/layers/ladder-down_module_list-0-conv1.bin",
|
||||
"shelfnet_berkeley/layers/ladder-down_module_list-0-conv12.bin",
|
||||
"shelfnet_berkeley/layers/ladder-down_conv_list-0.bin",
|
||||
|
||||
"shelfnet_berkeley/layers/ladder-down_module_list-1-conv1.bin",
|
||||
"shelfnet_berkeley/layers/ladder-down_module_list-1-conv12.bin",
|
||||
"shelfnet_berkeley/layers/ladder-down_conv_list-1.bin",
|
||||
|
||||
"shelfnet_berkeley/layers/ladder-bottom-conv1.bin",
|
||||
"shelfnet_berkeley/layers/ladder-bottom-conv12.bin",
|
||||
|
||||
|
||||
|
||||
"shelfnet_berkeley/layers/ladder-up_conv_list-0-conv-conv.bin",
|
||||
"shelfnet_berkeley/layers/ladder-up_conv_list-0-conv_atten.bin",
|
||||
"shelfnet_berkeley/layers/ladder-up_dense_list-0-conv.bin",
|
||||
|
||||
|
||||
"shelfnet_berkeley/layers/ladder-up_conv_list-1-conv-conv.bin",
|
||||
"shelfnet_berkeley/layers/ladder-up_conv_list-1-conv_atten.bin",
|
||||
"shelfnet_berkeley/layers/ladder-up_dense_list-1-conv.bin"};
|
||||
|
||||
const char *trans[] = {
|
||||
"shelfnet_berkeley/layers/trans1-conv.bin",
|
||||
"shelfnet_berkeley/layers/trans2-conv.bin",
|
||||
"shelfnet_berkeley/layers/trans3-conv.bin"};
|
||||
int main()
|
||||
{
|
||||
|
||||
downloadWeightsifDoNotExist(input_bin, "shelfnet_berkeley", "https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download");
|
||||
|
||||
int classes = 20;
|
||||
|
||||
// Network layout
|
||||
tk::dnn::dataDim_t dim(1, 3, 736, 1280, 1);
|
||||
tk::dnn::Network net(dim);
|
||||
|
||||
int bi = 0, di = 0, li = 0, ci = 0;
|
||||
new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX);
|
||||
|
||||
|
||||
|
||||
for(int i=0; i<2; ++i){
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
}
|
||||
|
||||
std::vector<tk::dnn::Layer*> features;
|
||||
for(int i=0;i<3;++i){
|
||||
int out_channel = pow(2,7+i);
|
||||
std::cout<<out_channel<<std::endl;
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 2, 2, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
tk::dnn::Layer* bn2 = new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Route(&net, &last, 1);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 2, 2, 0, 0, backbone[bi++], true);
|
||||
new tk::dnn::Shortcut(&net, bn2);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
features.push_back(last);
|
||||
}
|
||||
|
||||
for(int i=0; i<features.size(); ++i){
|
||||
new tk::dnn::Route(&net, &features[i], 1);
|
||||
int out_channel = pow(2,6+i);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, trans[i], true);
|
||||
features[i] = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
}
|
||||
|
||||
//DECODER
|
||||
|
||||
last = features[2];
|
||||
std::vector<tk::dnn::Layer*> up_out;
|
||||
//bottom
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
up_out.push_back(last);
|
||||
|
||||
for(int i=0; i<2; ++i){
|
||||
int out_channel = pow(2,7-i);
|
||||
//up-conv
|
||||
std::cout<<out_channel<<std::endl;
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
|
||||
new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true);
|
||||
|
||||
tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID);
|
||||
new tk::dnn::Route(&net, &last, 1);
|
||||
new tk::dnn::Shortcut(&net, act, true);
|
||||
|
||||
//interpolate
|
||||
new tk::dnn::Resize(&net, 1,2,2);
|
||||
new tk::dnn::Shortcut(&net, features[1-i]);
|
||||
|
||||
//up-dense
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
up_out.push_back(last);
|
||||
}
|
||||
|
||||
//LADDER
|
||||
|
||||
std::vector<tk::dnn::Layer*> down_out;
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
for(int i=0; i<2;++i){
|
||||
int out_channel = pow(2,6+i);
|
||||
tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]);
|
||||
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, l_last);
|
||||
l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
down_out.push_back(l_last);
|
||||
|
||||
new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU
|
||||
}
|
||||
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
up_out.clear();
|
||||
up_out.push_back(last);
|
||||
|
||||
for(int i=0; i<2; ++i){
|
||||
int out_channel = pow(2,7-i);
|
||||
//up-conv
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
|
||||
new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true);
|
||||
|
||||
tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID);
|
||||
new tk::dnn::Route(&net, &last, 1);
|
||||
new tk::dnn::Shortcut(&net, act, true);
|
||||
|
||||
//interpolate
|
||||
new tk::dnn::Resize(&net, 1,2,2);
|
||||
new tk::dnn::Shortcut(&net, down_out[1-i]);
|
||||
|
||||
// //up-dense
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
up_out.push_back(last);
|
||||
}
|
||||
|
||||
|
||||
// for(int i=2;i>=0;--i){
|
||||
// new tk::dnn::Route(&net, &up_out[i], 1);
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, classes, 3, 3, 1, 1, 1, 1, conv_out[ci++], false);
|
||||
/*up_out[i] =*/ new tk::dnn::Resize(&net, classes, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR);
|
||||
// }
|
||||
|
||||
new tk::dnn::Softmax(&net);
|
||||
|
||||
const char *output_bin = "shelfnet_berkeley/debug/softmax.bin";
|
||||
|
||||
// Load input
|
||||
dnnType *data;
|
||||
dnnType *input_h;
|
||||
readBinaryFile(input_bin, dim.tot(), &input_h, &data);
|
||||
std::cout<<"Input:"<<std::endl;
|
||||
|
||||
//print network model
|
||||
net.print();
|
||||
|
||||
// // convert network to tensorRT
|
||||
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("shelfnet_berkeley"));
|
||||
|
||||
tk::dnn::dataDim_t dim1 = dim; //input dim
|
||||
dnnType *cudnn_out = nullptr;
|
||||
printCenteredTitle(" CUDNN inference ", '=', 30);
|
||||
{
|
||||
dim1.print();
|
||||
TKDNN_TSTART
|
||||
cudnn_out = net.infer(dim1, data);
|
||||
TKDNN_TSTOP
|
||||
dim1.print();
|
||||
}
|
||||
|
||||
tk::dnn::dataDim_t dim2 = dim;
|
||||
printCenteredTitle(" TENSORRT inference ", '=', 30);
|
||||
{
|
||||
dim2.print();
|
||||
TKDNN_TSTART
|
||||
netRT.infer(dim2, data);
|
||||
TKDNN_TSTOP
|
||||
dim2.print();
|
||||
}
|
||||
|
||||
dnnType *rt_out1 = (dnnType *)netRT.buffersRT[1];
|
||||
|
||||
printCenteredTitle(std::string(" CHECK RESULTS ").c_str(), '=', 30);
|
||||
dnnType *out1, *out1_h;
|
||||
int odim1 = dim1.tot();
|
||||
readBinaryFile(output_bin, odim1, &out1_h, &out1);
|
||||
|
||||
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
|
||||
std::cout << "CUDNN vs correct" << std::endl;
|
||||
ret_cudnn |= checkResult(odim1, cudnn_out, out1, true, 20) == 0 ? 0 : ERROR_CUDNN;
|
||||
|
||||
std::cout << "TRT vs correct" << std::endl;
|
||||
ret_tensorrt |=checkResult(odim1, rt_out1, out1) == 0 ? 0 : ERROR_TENSORRT;
|
||||
|
||||
std::cout << "CUDNN vs TRT " << std::endl;
|
||||
ret_cudnn_tensorrt |= checkResult(odim1, cudnn_out, rt_out1) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
|
||||
|
||||
cv::Mat viz = vizLayer2Mat(&net, net.num_layers-1);
|
||||
cv::imwrite("test.png", viz);
|
||||
|
||||
return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
#include <iostream>
|
||||
#include <opencv2/highgui/highgui.hpp>
|
||||
#include <opencv2/imgproc/imgproc.hpp>
|
||||
|
||||
#include "tkdnn.h"
|
||||
#include "NetworkViz.h"
|
||||
|
||||
|
||||
const char *input_bin = "shelfnet_mapillary/debug/input.bin";
|
||||
|
||||
const char *backbone[] = {
|
||||
"shelfnet_mapillary/layers/backbone-conv1.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer1-0-conv1.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer1-0-conv2.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer1-1-conv1.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer1-1-conv2.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer2-0-conv1.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer2-0-conv2.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer2-0-downsample-0.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer2-1-conv1.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer2-1-conv2.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer3-0-conv1.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer3-0-conv2.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer3-0-downsample-0.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer3-1-conv1.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer3-1-conv2.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer4-0-conv1.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer4-0-conv2.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer4-0-downsample-0.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer4-1-conv1.bin",
|
||||
"shelfnet_mapillary/layers/backbone-layer4-1-conv2.bin"};
|
||||
|
||||
const char *conv_out[] = {
|
||||
"shelfnet_mapillary/layers/conv_out-conv-conv.bin",
|
||||
"shelfnet_mapillary/layers/conv_out-conv_out.bin",
|
||||
"shelfnet_mapillary/layers/conv_out16-conv-conv.bin",
|
||||
"shelfnet_mapillary/layers/conv_out16-conv_out.bin",
|
||||
"shelfnet_mapillary/layers/conv_out32-conv-conv.bin",
|
||||
"shelfnet_mapillary/layers/conv_out32-conv_out.bin"
|
||||
};
|
||||
|
||||
const char *decoder[] = {
|
||||
"shelfnet_mapillary/layers/decoder-bottom-conv1.bin",
|
||||
"shelfnet_mapillary/layers/decoder-bottom-conv12.bin",
|
||||
"shelfnet_mapillary/layers/decoder-up_conv_list-0-conv-conv.bin",
|
||||
"shelfnet_mapillary/layers/decoder-up_conv_list-0-conv_atten.bin",
|
||||
"shelfnet_mapillary/layers/decoder-up_dense_list-0-conv.bin",
|
||||
"shelfnet_mapillary/layers/decoder-up_conv_list-1-conv-conv.bin",
|
||||
"shelfnet_mapillary/layers/decoder-up_conv_list-1-conv_atten.bin",
|
||||
"shelfnet_mapillary/layers/decoder-up_dense_list-1-conv.bin"
|
||||
};
|
||||
|
||||
|
||||
const char *ladder[] = {
|
||||
"shelfnet_mapillary/layers/ladder-inconv-conv1.bin",
|
||||
"shelfnet_mapillary/layers/ladder-inconv-conv12.bin",
|
||||
"shelfnet_mapillary/layers/ladder-down_module_list-0-conv1.bin",
|
||||
"shelfnet_mapillary/layers/ladder-down_module_list-0-conv12.bin",
|
||||
"shelfnet_mapillary/layers/ladder-down_conv_list-0.bin",
|
||||
|
||||
"shelfnet_mapillary/layers/ladder-down_module_list-1-conv1.bin",
|
||||
"shelfnet_mapillary/layers/ladder-down_module_list-1-conv12.bin",
|
||||
"shelfnet_mapillary/layers/ladder-down_conv_list-1.bin",
|
||||
|
||||
"shelfnet_mapillary/layers/ladder-bottom-conv1.bin",
|
||||
"shelfnet_mapillary/layers/ladder-bottom-conv12.bin",
|
||||
|
||||
|
||||
|
||||
"shelfnet_mapillary/layers/ladder-up_conv_list-0-conv-conv.bin",
|
||||
"shelfnet_mapillary/layers/ladder-up_conv_list-0-conv_atten.bin",
|
||||
"shelfnet_mapillary/layers/ladder-up_dense_list-0-conv.bin",
|
||||
|
||||
|
||||
"shelfnet_mapillary/layers/ladder-up_conv_list-1-conv-conv.bin",
|
||||
"shelfnet_mapillary/layers/ladder-up_conv_list-1-conv_atten.bin",
|
||||
"shelfnet_mapillary/layers/ladder-up_dense_list-1-conv.bin"};
|
||||
|
||||
const char *trans[] = {
|
||||
"shelfnet_mapillary/layers/trans1-conv.bin",
|
||||
"shelfnet_mapillary/layers/trans2-conv.bin",
|
||||
"shelfnet_mapillary/layers/trans3-conv.bin"};
|
||||
int main()
|
||||
{
|
||||
|
||||
// downloadWeightsifDoNotExist(input_bin, "shelfnet_mapillary", "");
|
||||
// download the weights from here: https://cloud.hipert.unimore.it/f/652476
|
||||
|
||||
// Mapillary Vistas has originally 66 classes, but we reduced them to 15 to improve the results on the categories of our interest.
|
||||
int classes = 15;
|
||||
|
||||
// Network layout
|
||||
tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1);
|
||||
tk::dnn::Network net(dim);
|
||||
|
||||
int bi = 0, di = 0, li = 0, ci = 0;
|
||||
new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX);
|
||||
|
||||
|
||||
|
||||
for(int i=0; i<2; ++i){
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
}
|
||||
|
||||
std::vector<tk::dnn::Layer*> features;
|
||||
for(int i=0;i<3;++i){
|
||||
int out_channel = pow(2,7+i);
|
||||
std::cout<<out_channel<<std::endl;
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 2, 2, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
tk::dnn::Layer* bn2 = new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Route(&net, &last, 1);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 2, 2, 0, 0, backbone[bi++], true);
|
||||
new tk::dnn::Shortcut(&net, bn2);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
|
||||
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
features.push_back(last);
|
||||
}
|
||||
|
||||
for(int i=0; i<features.size(); ++i){
|
||||
new tk::dnn::Route(&net, &features[i], 1);
|
||||
int out_channel = pow(2,6+i);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, trans[i], true);
|
||||
features[i] = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
}
|
||||
|
||||
//DECODER
|
||||
|
||||
last = features[2];
|
||||
std::vector<tk::dnn::Layer*> up_out;
|
||||
//bottom
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
up_out.push_back(last);
|
||||
|
||||
for(int i=0; i<2; ++i){
|
||||
int out_channel = pow(2,7-i);
|
||||
//up-conv
|
||||
std::cout<<out_channel<<std::endl;
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
|
||||
new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true);
|
||||
|
||||
tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID);
|
||||
new tk::dnn::Route(&net, &last, 1);
|
||||
new tk::dnn::Shortcut(&net, act, true);
|
||||
|
||||
//interpolate
|
||||
new tk::dnn::Resize(&net, 1,2,2);
|
||||
new tk::dnn::Shortcut(&net, features[1-i]);
|
||||
|
||||
//up-dense
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
up_out.push_back(last);
|
||||
}
|
||||
|
||||
//LADDER
|
||||
|
||||
std::vector<tk::dnn::Layer*> down_out;
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
|
||||
for(int i=0; i<2;++i){
|
||||
int out_channel = pow(2,6+i);
|
||||
tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]);
|
||||
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, l_last);
|
||||
l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
down_out.push_back(l_last);
|
||||
|
||||
new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU
|
||||
}
|
||||
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
|
||||
new tk::dnn::Shortcut(&net, last);
|
||||
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
|
||||
up_out.clear();
|
||||
up_out.push_back(last);
|
||||
|
||||
for(int i=0; i<2; ++i){
|
||||
int out_channel = pow(2,7-i);
|
||||
//up-conv
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
|
||||
new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE);
|
||||
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true);
|
||||
|
||||
tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID);
|
||||
new tk::dnn::Route(&net, &last, 1);
|
||||
new tk::dnn::Shortcut(&net, act, true);
|
||||
|
||||
//interpolate
|
||||
new tk::dnn::Resize(&net, 1,2,2);
|
||||
new tk::dnn::Shortcut(&net, down_out[1-i]);
|
||||
|
||||
// //up-dense
|
||||
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true);
|
||||
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
up_out.push_back(last);
|
||||
}
|
||||
|
||||
|
||||
// for(int i=2;i>=0;--i){
|
||||
// new tk::dnn::Route(&net, &up_out[i], 1);
|
||||
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true);
|
||||
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
|
||||
new tk::dnn::Conv2d (&net, classes, 3, 3, 1, 1, 1, 1, conv_out[ci++], false);
|
||||
/*up_out[i] =*/ new tk::dnn::Resize(&net, classes, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR);
|
||||
// }
|
||||
|
||||
new tk::dnn::Softmax(&net);
|
||||
|
||||
const char *output_bin = "shelfnet_mapillary/debug/softmax.bin";
|
||||
|
||||
// Load input
|
||||
dnnType *data;
|
||||
dnnType *input_h;
|
||||
readBinaryFile(input_bin, dim.tot(), &input_h, &data);
|
||||
std::cout<<"Input:"<<std::endl;
|
||||
|
||||
//print network model
|
||||
net.print();
|
||||
|
||||
// // convert network to tensorRT
|
||||
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("shelfnet_mapillary"));
|
||||
|
||||
tk::dnn::dataDim_t dim1 = dim; //input dim
|
||||
dnnType *cudnn_out = nullptr;
|
||||
printCenteredTitle(" CUDNN inference ", '=', 30);
|
||||
{
|
||||
dim1.print();
|
||||
TKDNN_TSTART
|
||||
cudnn_out = net.infer(dim1, data);
|
||||
TKDNN_TSTOP
|
||||
dim1.print();
|
||||
}
|
||||
|
||||
tk::dnn::dataDim_t dim2 = dim;
|
||||
printCenteredTitle(" TENSORRT inference ", '=', 30);
|
||||
{
|
||||
dim2.print();
|
||||
TKDNN_TSTART
|
||||
netRT.infer(dim2, data);
|
||||
TKDNN_TSTOP
|
||||
dim2.print();
|
||||
}
|
||||
|
||||
dnnType *rt_out1 = (dnnType *)netRT.buffersRT[1];
|
||||
|
||||
printCenteredTitle(std::string(" CHECK RESULTS ").c_str(), '=', 30);
|
||||
dnnType *out1, *out1_h;
|
||||
int odim1 = dim1.tot();
|
||||
readBinaryFile(output_bin, odim1, &out1_h, &out1);
|
||||
|
||||
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
|
||||
std::cout << "CUDNN vs correct" << std::endl;
|
||||
ret_cudnn |= checkResult(odim1, cudnn_out, out1, true, 20) == 0 ? 0 : ERROR_CUDNN;
|
||||
|
||||
std::cout << "TRT vs correct" << std::endl;
|
||||
ret_tensorrt |=checkResult(odim1, rt_out1, out1) == 0 ? 0 : ERROR_TENSORRT;
|
||||
|
||||
std::cout << "CUDNN vs TRT " << std::endl;
|
||||
ret_cudnn_tensorrt |= checkResult(odim1, cudnn_out, rt_out1) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
|
||||
|
||||
cv::Mat viz = vizLayer2Mat(&net, net.num_layers-1);
|
||||
cv::imwrite("test.png", viz);
|
||||
|
||||
return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
|
||||
}
|
||||
@@ -31,7 +31,7 @@ int main(int argc, char *argv[]) {
|
||||
std::cout<<"Testing with batchsize: "<<BATCH_SIZE<<"\n";
|
||||
printCenteredTitle(" TENSORRT inference ", '=', 30);
|
||||
float total_time = 0;
|
||||
for(int i=0; i<1200; i++) {
|
||||
for(int i=0; i<64; i++) {
|
||||
|
||||
// generate input
|
||||
for(int j=0; j<netRT.input_dim.tot(); j++) {
|
||||
@@ -58,6 +58,6 @@ int main(int argc, char *argv[]) {
|
||||
}
|
||||
}
|
||||
}
|
||||
std::cout<<"avg: "<<total_time/1200.<<std::endl;
|
||||
std::cout<<"avg: "<<total_time/64.<<std::endl;
|
||||
return ret_tensorrt;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user