mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

Add shelfnet_mapillary, README_seg, resize of input

Browse Source 
Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>
This commit is contained in:
Micaela Verucchi
2020-11-24 12:37:33 +01:00
parent 286e777300
commit a52e18b6e6
7 changed files with 498 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+3
View File
@@ -110,6 +110,9 @@ target_link_libraries(test_shelfnet tkDNN)
add_executable(test_shelfnet_berkeley tests/shelfnet/shelfnet_berkeley.cpp)
target_link_libraries(test_shelfnet_berkeley tkDNN)
add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp)
target_link_libraries(test_shelfnet_mapillary tkDNN)
# DEMOS
add_executable(test_rtinference tests/test_rtinference/rtinference.cpp)
target_link_libraries(test_rtinference tkDNN)
README_seg.md
+56
View File
@@ -0,0 +1,56 @@
# Semantic Segmentation with tkDNN
Currently tkDNN supports only ShelfNet as semantic segmentation network.
## Export weights from Shelfnet
To get the weights needed to run Mobilenet tests use [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of a Pytorch implementation of Shelfnet network.
```
git clone https://git.hipert.unimore.it/mverucchi/shelfnet
cd shelfnet
cd ShelfNet18_realtime
conda env create --file shelfnet_env.yml
conda activate shelfnet
mkdir layer debug
python export.py
```
## Run the demo
To run the semantic segmentation demo follow these steps (example with shelfnet_mapillary):
```
rm shelfnet_mapillary_fp32.rt # be sure to delete(or move) old tensorRT files
export TKDNN_BATCHSIZE=4 # be sure you have batch size > than 1 if you want to run inference on images bigger than 1024
./test_shelfnet_mapillary # run the yolo test (is slow)
./demo shelfnet_mapillary_fp32.rt ../demo/yolo_test.mp4 1 15
```
In general the demo program takes the following parameters:
```
./seg_demo <network-rt-file> <path-to-video> <n-batches> <number-of-classes> <resize-flag> <baseline-resize> <show-flag> <write-pred>
```
where
* ```<network-rt-file>``` is the rt file generated by a test
* ```<<path-to-video>``` is the path to a video file or a camera input
* ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
* ```<number-of-classes>```is the number of classes the network is trained on
* ```<resize-flag>``` if set to 0 the demo will not resize the input frames, but use it as it is, otherwise it will resize it.
* ```<baseline-resize>``` is ```<resize-flag>``` is set to 1, then the input frames will be proportionally resized using ```<baseline-resize>``` as width baseline.
* ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
* ```<write-pred>``` if set to 0 (deafult) the demo will run, otherwise the evaluation of a dataset will run and the output of the segmentation will be saved. Attention: this is under development and paths are embedded, so change them in the code in advance.
N.b. By default it is used FP32 inference
<!-- TODO: add gif -->
## Existing tests and supported networks
| Test Name | Network | Dataset | N Classes | Input size | Weights |
| :---------------- | :-------------------------------------------- | :-----------------------------------------------------------: | :-------: | :-----------: | :------------------------------------------------------------------------ |
| shelfnet | ShelfNet18_realtime<sup>1</sup> | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) |
| shelfnet_berkeley | ShelfNet18_realtime<sup>1</sup> | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) |
| shelfnet_mapillary | ShelfNet18_realtime<sup>1</sup> | [Mapillary Vistas](https://www.mapillary.com/dataset/vistas?pKey=aFWuj_m4nGoq3-tDz5KAqQ)<sup>*</sup> | 15 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/6WnZCKLjik7xrny/download) |
1. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019.
*. Mapillary Vistas has originally 66 classes, but we reduced them to 15 to improve the results on the categories of our interest.
demo/demo/seg_demo.cpp
+38 -10
View File
@@ -48,22 +48,37 @@ int main(int argc, char *argv[]) {
int n_classes = 19;
if(argc > 4)
n_classes = atoi(argv[4]);
bool show = true;
bool resize = false;
if(argc > 5)
show = atoi(argv[5]);
bool write_pred = false;
resize = atoi(argv[5]);
int baseline_resize = 1024;
if(argc > 6)
write_pred = atoi(argv[6]);
baseline_resize = atoi(argv[6]);
bool show = true;
if(argc > 7)
show = atoi(argv[7]);
bool write_pred = false;
if(argc > 8)
write_pred = atoi(argv[8]);
if(resize && (baseline_resize < 0 || baseline_resize > 5000))
FatalError("Problem with baseline resize")
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
std::string net_name;
removePathAndExtension(net, net_name);
bool mapillary_15 = false; //TODO change me pls
if(n_classes == 15 && net_name == "shelfnet_mapillary_fp32")
mapillary_15 = true;
//net initialization
tk::dnn::SegmentationNN segNN;
segNN.init(net, n_classes, n_batch);
int height = 0, width = 0;
int basewidth=baseline_resize, hsize;
if(write_pred){
std::string gt_folder = "../demo/CityScapes_val/images/";
std::string images_names = "../demo/CityScapes_val/all_images.txt";
@@ -85,9 +100,16 @@ int main(int argc, char *argv[]) {
cv::VideoWriter resultVideo;
if(SAVE_RESULT) {
int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(1024, 1024));
int w,h;
if(resize){
w = basewidth;
h = int((float(cap.get(cv::CAP_PROP_FRAME_HEIGHT))*float(basewidth/float(cap.get(cv::CAP_PROP_FRAME_WIDTH)))));
}
else{
w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
}
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
cv::Mat frame;
@@ -95,11 +117,17 @@ int main(int argc, char *argv[]) {
cap >> frame;
if(!frame.data)
break;
if(resize){
hsize = int((float(frame.rows)*float(basewidth/float(frame.cols))));
cv::resize(frame, frame, cv::Size(basewidth, hsize));
}
height = frame.rows;
width = frame.cols;
//inference
segNN.updateOriginal(frame);
segNN.updateOriginal(frame, true, mapillary_15);
if(show)
segNN.draw();
include/tkDNN/NetworkViz.h
+2 -2
View File
@@ -5,8 +5,8 @@
namespace tk { namespace dnn {
cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0);
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, double min=0, double max=0);
cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, bool mapillary_15=false);
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, double min=0, double max=0, bool mapillary_15=false);
cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000);
}}
include/tkDNN/SegmentationNN.h
+4 -4
View File
@@ -97,7 +97,7 @@ class SegmentationNN {
*
* @param bi batch index
*/
void postprocess(const int bi=0, bool appy_colormap = true) {
void postprocess(const int bi=0, bool appy_colormap = true, bool mapillary_15=false) {
dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
dataDim_t odim = netRT->output_dim;
@@ -112,7 +112,7 @@ class SegmentationNN {
cv::Mat colored;
if(appy_colormap)
colored = vizData2Mat(tmpOutData_h, vdim, 1024, 0, 18);
colored = vizData2Mat(tmpOutData_h, vdim, 1024, 0, classes, mapillary_15);
else{
cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h);
colored_fp32.convertTo(colored, CV_8UC1);
@@ -234,7 +234,7 @@ class SegmentationNN {
}
}
void updateOriginal(cv::Mat frame, bool apply_colormap=true){
void updateOriginal(cv::Mat frame, bool apply_colormap=true, bool mapillary_15=false){
std::vector<cv::Mat> splitted_frames;
int H, W, net_H, net_W;
@@ -347,7 +347,7 @@ class SegmentationNN {
cv::Mat colored;
if(apply_colormap)
colored = vizData2Mat(tmpOutData_h, vdim, 1024, 0, 18);
colored = vizData2Mat(tmpOutData_h, vdim, 1024, 0, classes, mapillary_15);
else{
cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h);
colored_fp32.convertTo(colored, CV_8UC1);
src/NetworkViz.cpp
+100 -8
View File
@@ -6,22 +6,114 @@
namespace tk { namespace dnn {
cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max) {
cv::Mat mapillary_15_map(cv::Mat adjMap){
// cv::imshow("test", adjMap);
// cv::waitKey(0);
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
M3.at<uchar>(0)=165;
M2.at<uchar>(0)=42;
M1.at<uchar>(0)=45;
M3.at<uchar>(1)=196;
M2.at<uchar>(1)=196;
M1.at<uchar>(1)=196;
M3.at<uchar>(2)=90;
M2.at<uchar>(2)=120;
M1.at<uchar>(2)=150;
M3.at<uchar>(3)=128;
M2.at<uchar>(3)=64;
M1.at<uchar>(3)=128;
M3.at<uchar>(4)=70;
M2.at<uchar>(4)=70;
M1.at<uchar>(4)=70;
M3.at<uchar>(5)=220;
M2.at<uchar>(5)=20;
M1.at<uchar>(5)=60;
M3.at<uchar>(6)=255;
M2.at<uchar>(6)=255;
M1.at<uchar>(6)=255;
M3.at<uchar>(7)=107;
M2.at<uchar>(7)=142;
M1.at<uchar>(7)=35;
M3.at<uchar>(8)=70;
M2.at<uchar>(8)=130;
M1.at<uchar>(8)=180;
M3.at<uchar>(9)=220;
M2.at<uchar>(9)=220;
M1.at<uchar>(9)=220;
M3.at<uchar>(10)=153;
M2.at<uchar>(10)=153;
M1.at<uchar>(10)=153;
M3.at<uchar>(11)=128;
M2.at<uchar>(11)=128;
M1.at<uchar>(11)=128;
M3.at<uchar>(12)=119;
M2.at<uchar>(12)=11;
M1.at<uchar>(12)=32;
M3.at<uchar>(13)=0;
M2.at<uchar>(13)=0;
M1.at<uchar>(13)=142;
for(int i=14;i<256;i++)
{
M1.at<uchar>(i)=0;
M2.at<uchar>(i)=0;
M3.at<uchar>(i)=0;
}
cv::Mat r1,r2,r3;
cv::LUT(adjMap,M1,r1);
cv::LUT(adjMap,M2,r2);
cv::LUT(adjMap,M3,r3);
std::vector<cv::Mat> planes;
planes.push_back(r1);
planes.push_back(r2);
planes.push_back(r3);
cv::Mat dst;
cv::merge(planes,dst);
return dst;
}
cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, bool mapillary_15) {
if(min == 0 && max == 0)
cv::minMaxIdx(map, &min, &max);
cv::Mat adjMap;
// expand your range to 0..255. Similar to histEq();
map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min);
//return adjMap;
cv::Mat falseColorsMap;
applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_VIRIDIS);
if(mapillary_15){
map.convertTo(adjMap,CV_8UC1);
falseColorsMap = mapillary_15_map(adjMap);
}
else{
// expand your range to 0..255. Similar to histEq();
map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min);
applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_JET);
}
return falseColorsMap;
}
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, double min, double max) {
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, double min, double max, bool mapillary_15) {
dnnType *data = nullptr;
// copy to CPU
@@ -37,7 +129,7 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim, doub
cv::Mat grid = cv::Mat(gridSize, CV_8UC3, cv::Scalar(0));
for(int i=0; i<dim.c;i++) {
cv::Mat raw = vizFloat2colorMap(cv::Mat(cv::Size(dim.w, dim.h),CV_32FC1, data + dim.w*dim.h*i), min, max);
cv::Mat raw = vizFloat2colorMap(cv::Mat(cv::Size(dim.w, dim.h),CV_32FC1, data + dim.w*dim.h*i), min, max, mapillary_15);
int r = i / gridDim;
int c = i - r * gridDim;
raw.copyTo(grid.rowRange(r*dim.h, r*dim.h + dim.h).colRange(c*dim.w, c*dim.w + dim.w));
tests/shelfnet/shelfnet_mapillary.cpp
+295
View File
@@ -0,0 +1,295 @@
#include <iostream>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "tkdnn.h"
#include "NetworkViz.h"
const char *input_bin = "shelfnet_mapillary/debug/input.bin";
const char *backbone[] = {
"shelfnet_mapillary/layers/backbone-conv1.bin",
"shelfnet_mapillary/layers/backbone-layer1-0-conv1.bin",
"shelfnet_mapillary/layers/backbone-layer1-0-conv2.bin",
"shelfnet_mapillary/layers/backbone-layer1-1-conv1.bin",
"shelfnet_mapillary/layers/backbone-layer1-1-conv2.bin",
"shelfnet_mapillary/layers/backbone-layer2-0-conv1.bin",
"shelfnet_mapillary/layers/backbone-layer2-0-conv2.bin",
"shelfnet_mapillary/layers/backbone-layer2-0-downsample-0.bin",
"shelfnet_mapillary/layers/backbone-layer2-1-conv1.bin",
"shelfnet_mapillary/layers/backbone-layer2-1-conv2.bin",
"shelfnet_mapillary/layers/backbone-layer3-0-conv1.bin",
"shelfnet_mapillary/layers/backbone-layer3-0-conv2.bin",
"shelfnet_mapillary/layers/backbone-layer3-0-downsample-0.bin",
"shelfnet_mapillary/layers/backbone-layer3-1-conv1.bin",
"shelfnet_mapillary/layers/backbone-layer3-1-conv2.bin",
"shelfnet_mapillary/layers/backbone-layer4-0-conv1.bin",
"shelfnet_mapillary/layers/backbone-layer4-0-conv2.bin",
"shelfnet_mapillary/layers/backbone-layer4-0-downsample-0.bin",
"shelfnet_mapillary/layers/backbone-layer4-1-conv1.bin",
"shelfnet_mapillary/layers/backbone-layer4-1-conv2.bin"};
const char *conv_out[] = {
"shelfnet_mapillary/layers/conv_out-conv-conv.bin",
"shelfnet_mapillary/layers/conv_out-conv_out.bin",
"shelfnet_mapillary/layers/conv_out16-conv-conv.bin",
"shelfnet_mapillary/layers/conv_out16-conv_out.bin",
"shelfnet_mapillary/layers/conv_out32-conv-conv.bin",
"shelfnet_mapillary/layers/conv_out32-conv_out.bin"
};
const char *decoder[] = {
"shelfnet_mapillary/layers/decoder-bottom-conv1.bin",
"shelfnet_mapillary/layers/decoder-bottom-conv12.bin",
"shelfnet_mapillary/layers/decoder-up_conv_list-0-conv-conv.bin",
"shelfnet_mapillary/layers/decoder-up_conv_list-0-conv_atten.bin",
"shelfnet_mapillary/layers/decoder-up_dense_list-0-conv.bin",
"shelfnet_mapillary/layers/decoder-up_conv_list-1-conv-conv.bin",
"shelfnet_mapillary/layers/decoder-up_conv_list-1-conv_atten.bin",
"shelfnet_mapillary/layers/decoder-up_dense_list-1-conv.bin"
};
const char *ladder[] = {
"shelfnet_mapillary/layers/ladder-inconv-conv1.bin",
"shelfnet_mapillary/layers/ladder-inconv-conv12.bin",
"shelfnet_mapillary/layers/ladder-down_module_list-0-conv1.bin",
"shelfnet_mapillary/layers/ladder-down_module_list-0-conv12.bin",
"shelfnet_mapillary/layers/ladder-down_conv_list-0.bin",
"shelfnet_mapillary/layers/ladder-down_module_list-1-conv1.bin",
"shelfnet_mapillary/layers/ladder-down_module_list-1-conv12.bin",
"shelfnet_mapillary/layers/ladder-down_conv_list-1.bin",
"shelfnet_mapillary/layers/ladder-bottom-conv1.bin",
"shelfnet_mapillary/layers/ladder-bottom-conv12.bin",
"shelfnet_mapillary/layers/ladder-up_conv_list-0-conv-conv.bin",
"shelfnet_mapillary/layers/ladder-up_conv_list-0-conv_atten.bin",
"shelfnet_mapillary/layers/ladder-up_dense_list-0-conv.bin",
"shelfnet_mapillary/layers/ladder-up_conv_list-1-conv-conv.bin",
"shelfnet_mapillary/layers/ladder-up_conv_list-1-conv_atten.bin",
"shelfnet_mapillary/layers/ladder-up_dense_list-1-conv.bin"};
const char *trans[] = {
"shelfnet_mapillary/layers/trans1-conv.bin",
"shelfnet_mapillary/layers/trans2-conv.bin",
"shelfnet_mapillary/layers/trans3-conv.bin"};
int main()
{
downloadWeightsifDoNotExist(input_bin, "shelfnet_mapillary", "https://cloud.hipert.unimore.it/s/6WnZCKLjik7xrny/download");
int classes = 15;
// Network layout
tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1);
tk::dnn::Network net(dim);
int bi = 0, di = 0, li = 0, ci = 0;
new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true);
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX);
for(int i=0; i<2; ++i){
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
new tk::dnn::Shortcut(&net, last);
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
}
std::vector<tk::dnn::Layer*> features;
for(int i=0;i<3;++i){
int out_channel = pow(2,7+i);
std::cout<<out_channel<<std::endl;
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 2, 2, 1, 1, backbone[bi++], true);
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
tk::dnn::Layer* bn2 = new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
new tk::dnn::Route(&net, &last, 1);
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 2, 2, 0, 0, backbone[bi++], true);
new tk::dnn::Shortcut(&net, bn2);
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, backbone[bi++], true);
new tk::dnn::Shortcut(&net, last);
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
features.push_back(last);
}
for(int i=0; i<features.size(); ++i){
new tk::dnn::Route(&net, &features[i], 1);
int out_channel = pow(2,6+i);
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, trans[i], true);
features[i] = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
}
//DECODER
last = features[2];
std::vector<tk::dnn::Layer*> up_out;
//bottom
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true);
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true);
new tk::dnn::Shortcut(&net, last);
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
up_out.push_back(last);
for(int i=0; i<2; ++i){
int out_channel = pow(2,7-i);
//up-conv
std::cout<<out_channel<<std::endl;
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true);
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE);
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true);
tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID);
new tk::dnn::Route(&net, &last, 1);
new tk::dnn::Shortcut(&net, act, true);
//interpolate
new tk::dnn::Resize(&net, 1,2,2);
new tk::dnn::Shortcut(&net, features[1-i]);
//up-dense
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true);
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
up_out.push_back(last);
}
//LADDER
std::vector<tk::dnn::Layer*> down_out;
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
new tk::dnn::Shortcut(&net, last);
new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
for(int i=0; i<2;++i){
int out_channel = pow(2,6+i);
tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]);
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
new tk::dnn::Shortcut(&net, l_last);
l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
down_out.push_back(l_last);
new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false);
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU
}
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true);
new tk::dnn::Shortcut(&net, last);
last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU);
up_out.clear();
up_out.push_back(last);
for(int i=0; i<2; ++i){
int out_channel = pow(2,7-i);
//up-conv
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true);
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE);
new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true);
tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID);
new tk::dnn::Route(&net, &last, 1);
new tk::dnn::Shortcut(&net, act, true);
//interpolate
new tk::dnn::Resize(&net, 1,2,2);
new tk::dnn::Shortcut(&net, down_out[1-i]);
// //up-dense
new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true);
last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
up_out.push_back(last);
}
// for(int i=2;i>=0;--i){
// new tk::dnn::Route(&net, &up_out[i], 1);
new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true);
new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01);
new tk::dnn::Conv2d (&net, classes, 3, 3, 1, 1, 1, 1, conv_out[ci++], false);
/*up_out[i] =*/ new tk::dnn::Resize(&net, classes, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR);
// }
new tk::dnn::Softmax(&net);
const char *output_bin = "shelfnet_mapillary/debug/softmax.bin";
// Load input
dnnType *data;
dnnType *input_h;
readBinaryFile(input_bin, dim.tot(), &input_h, &data);
std::cout<<"Input:"<<std::endl;
//print network model
net.print();
// // convert network to tensorRT
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("shelfnet_mapillary"));
tk::dnn::dataDim_t dim1 = dim; //input dim
dnnType *cudnn_out = nullptr;
printCenteredTitle(" CUDNN inference ", '=', 30);
{
dim1.print();
TKDNN_TSTART
cudnn_out = net.infer(dim1, data);
TKDNN_TSTOP
dim1.print();
}
tk::dnn::dataDim_t dim2 = dim;
printCenteredTitle(" TENSORRT inference ", '=', 30);
{
dim2.print();
TKDNN_TSTART
netRT.infer(dim2, data);
TKDNN_TSTOP
dim2.print();
}
dnnType *rt_out1 = (dnnType *)netRT.buffersRT[1];
printCenteredTitle(std::string(" CHECK RESULTS ").c_str(), '=', 30);
dnnType *out1, *out1_h;
int odim1 = dim1.tot();
readBinaryFile(output_bin, odim1, &out1_h, &out1);
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
std::cout << "CUDNN vs correct" << std::endl;
ret_cudnn |= checkResult(odim1, cudnn_out, out1, true, 20) == 0 ? 0 : ERROR_CUDNN;
std::cout << "TRT vs correct" << std::endl;
ret_tensorrt |=checkResult(odim1, rt_out1, out1) == 0 ? 0 : ERROR_TENSORRT;
std::cout << "CUDNN vs TRT " << std::endl;
ret_cudnn_tensorrt |= checkResult(odim1, cudnn_out, rt_out1) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
cv::Mat viz = vizLayer2Mat(&net, net.num_layers-1);
cv::imwrite("test.png", viz);
return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
}