mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

fp16 implementation, TODO deallocate in LayerWgs

Browse Source
This commit is contained in:
Francesco Gatti
2017-08-30 14:37:25 +00:00
parent a26ef98d2d
commit 2cf8d8f6fc
11 changed files with 190 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/LayerWgs.cpp
+68
View File
@@ -1,6 +1,8 @@
#include <iostream>
#include <string.h>
#include "Layer.h"
#include "kernels.h"
namespace tkDNN {
@@ -26,6 +28,72 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek);
seek += outputs;
readBinaryFile(weights_path.c_str(), outputs, &variance_h, &variance_d, seek);
float eps = CUDNN_BN_MIN_EPSILON;
power_h = new dnnType[outputs];
for(int i=0; i<outputs; i++) power_h[i] = 1.0f;
for(int i=0; i<outputs; i++)
mean_h[i] = mean_h[i] / -sqrt(eps + variance_h[i]);
for(int i=0; i<outputs; i++)
variance_h[i] = 1.0f / sqrt(eps + variance_h[i]);
}
if(!net->fp16)
return;
//convert to fp16
int w_size = inputs*outputs*kh*kw*kl;
data16_h = new __half[w_size];
cudaMalloc(&data16_d, w_size*sizeof(__half));
float2half(data_d, data16_d, w_size);
cudaMemcpy(data16_h, data16_d, w_size*sizeof(__half), cudaMemcpyDeviceToHost);
int b_size = outputs;
bias16_h = new __half[b_size];
cudaMalloc(&bias16_d, w_size*sizeof(__half));
float2half(bias_d, bias16_d, b_size);
cudaMemcpy(bias16_h, bias16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
if(batchnorm) {
power16_h = new __half[b_size];
mean16_h = new __half[b_size];
variance16_h = new __half[b_size];
scales16_h = new __half[b_size];
cudaMalloc(&power16_d, b_size*sizeof(__half));
cudaMalloc(&mean16_d, b_size*sizeof(__half));
cudaMalloc(&variance16_d, b_size*sizeof(__half));
cudaMalloc(&scales16_d, b_size*sizeof(__half));
//temporary buffers
float *tmp_d;
cudaMalloc(&tmp_d, b_size*sizeof(float));
//init power array of ones
cudaMemcpy(tmp_d, power_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
float2half(tmp_d, power16_d, b_size);
cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
//mean array
cudaMemcpy(tmp_d, mean_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
float2half(tmp_d, mean16_d, b_size);
cudaMemcpy(mean16_h, mean16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
//convert variance
cudaMemcpy(tmp_d, variance_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
float2half(tmp_d, variance16_d, b_size);
cudaMemcpy(variance16_h, variance16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
//conver scales
float2half(scales_d, scales16_d, b_size);
cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
}
}
src/Network.cpp
+9 -1
View File
@@ -1,5 +1,5 @@
#include <iostream>
#include <string>
#include <string.h>
#include "tkdnn.h"
#include "Network.h"
@@ -22,6 +22,14 @@ Network::Network(dataDim_t input_dim) {
checkERROR( cublasCreate(&cublasHandle) );
num_layers = 0;
fp16 = false;
if(const char* env_p = std::getenv("TKDNN_MODE"))
if(strcmp(env_p, "FP16") == 0)
fp16 = true;
if(fp16)
std::cout<<COL_REDB<<"!! FP16 INERENCE ENABLED !!"<<COL_END<<"\n";
}
Network::~Network() {
src/NetworkRT.cpp
+45 -28
View File
@@ -1,9 +1,13 @@
#include <iostream>
#include <map>
#include <errno.h>
#include <string.h> // memcpy
#include <stdlib.h>
#include "kernels.h"
#include "utils.h"
#include "NvInfer.h"
#include "NetworkRT.h"
using namespace nvinfer1;
@@ -45,7 +49,7 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
builderRT->setMaxBatchSize(1);
builderRT->setMaxWorkspaceSize(1 << 30);
/*
//change datatype based on system specs
if(builderRT->platformHasFastInt8()) {
BatchStream bstream({32,dim.c, dim.h, dim.w}, 32, 1);
@@ -53,13 +57,13 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
builderRT->setInt8Mode(true);
builderRT->setInt8Calibrator(&calib);
} else if(builderRT->platformHasFastFp16()) {
} else if(net->fp16 && builderRT->platformHasFastFp16()) {
dtRT = DataType::kHALF;
builderRT->setHalf2Mode(true);
}
*/
//add input layer
ITensor *input = networkRT->addInput("data", dtRT,
ITensor *input = networkRT->addInput("data", DataType::kFLOAT,
DimsCHW{ dim.c, dim.h, dim.w});
checkNULL(input);
@@ -170,22 +174,49 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) {
ILayer* NetworkRT::convert_layer(ITensor *input, Dense *l) {
//std::cout<<"convert Dense\n";
void *data_b, *bias_b;
if(dtRT == DataType::kHALF) {
data_b = l->data16_h;
bias_b = l->bias16_h;
} else {
data_b = l->data_h;
bias_b = l->bias_h;
}
Weights w { dtRT, l->data_h, l->inputs*l->outputs};
Weights b = { dtRT, l->bias_h, l->outputs};
Weights w { dtRT, data_b, l->inputs*l->outputs};
Weights b = { dtRT, bias_b, l->outputs};
IFullyConnectedLayer *lRT = networkRT->addFullyConnected(*input, l->outputs, w, b);
checkNULL(lRT);
return lRT;
}
ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) {
//std::cout<<"convert conv2D\n";
Weights w { dtRT, l->data_h, l->inputs*l->outputs*l->kernelH*l->kernelW};
void *data_b, *bias_b, *power_b, *mean_b, *variance_b, *scales_b;
if(dtRT == DataType::kHALF) {
data_b = l->data16_h;
bias_b = l->bias16_h;
power_b = l->power16_h;
mean_b = l->mean16_h;
variance_b = l->variance16_h;
scales_b = l->scales16_h;
} else {
data_b = l->data_h;
bias_b = l->bias_h;
power_b = l->power_h;
mean_b = l->mean_h;
variance_b = l->variance_h;
scales_b = l->scales_h;
}
Weights w { dtRT, data_b, l->inputs*l->outputs*l->kernelH*l->kernelW};
Weights b;
if(!l->batchnorm)
b = { dtRT, l->bias_h, l->outputs};
b = { dtRT, bias_b, l->outputs};
else
b = { dtRT, nullptr, 0}; //on batchnorm bias are added later
@@ -198,29 +229,15 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) {
lRT->setPadding(DimsHW{l->paddingH, l->paddingW});
if(l->batchnorm) {
float eps = CUDNN_BN_MIN_EPSILON;
//make power array of ones
dnnType *power_h = new dnnType[l->outputs];
for(int i=0; i<l->outputs; i++) power_h[i] = 1.0f;
//convert mean
for(int i=0; i<l->outputs; i++)
l->mean_h[i] = l->mean_h[i] / -sqrt(eps + l->variance_h[i]);
//convert variance
for(int i=0; i<l->outputs; i++)
l->variance_h[i] = 1.0f / sqrt(eps + l->variance_h[i]);
Weights power{dtRT, power_h, l->outputs};
Weights shift{dtRT, l->mean_h, l->outputs};
Weights scale{dtRT, l->variance_h, l->outputs};
Weights power{dtRT, power_b, l->outputs};
Weights shift{dtRT, mean_b, l->outputs};
Weights scale{dtRT, variance_b, l->outputs};
IScaleLayer *lRT2 = networkRT->addScale(*lRT->getOutput(0), ScaleMode::kCHANNEL,
shift, scale, power);
checkNULL(lRT2);
Weights shift2{dtRT, l->bias_h, l->outputs};
Weights scale2{dtRT, l->scales_h, l->outputs};
Weights shift2{dtRT, bias_b, l->outputs};
Weights scale2{dtRT, scales_b, l->outputs};
IScaleLayer *lRT3 = networkRT->addScale(*lRT2->getOutput(0), ScaleMode::kCHANNEL,
shift2, scale2, power);
checkNULL(lRT3);
src/kernels/convert.cu
+21
View File
@@ -0,0 +1,21 @@
#include "kernels.h"
__global__
void float2half_device(float *input, __half *output, int size) {
int i = blockDim.x*blockIdx.x + threadIdx.x;
if(i<size) {
output[i] = __float2half(input[i]);
}
}
void float2half(float* srcData, __half *dstData, int size, const cudaStream_t stream)
{
int blocks = (size+255)/256;
int threads = 256;
float2half_device<<<blocks, threads, 0, stream>>>(srcData, dstData, size);
cudaDeviceSynchronize();
}
src/utils.cpp
+1 -1
View File
@@ -70,7 +70,7 @@ void printDeviceVector(int size, dnnType* vec_d, bool device)
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device) {
dnnType *data_h, *correct_h;
const float eps = 0.0001f;
const float eps = 0.001f;
if(device) {
data_h = new dnnType[size];