From 57c9a6ec991fb3bd51d67817bc15986efd1698c4 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Fri, 11 Aug 2017 16:32:22 +0200 Subject: [PATCH] LEAKY serialized --- include/NetworkRT.h | 34 ++++-- include/utils.h | 1 + src/NetworkRT.cpp | 165 +++++++++++++++++++--------- src/pluginsRT/ActivationLeakyRT.cpp | 4 +- src/utils.cpp | 5 + 5 files changed, 150 insertions(+), 59 deletions(-) diff --git a/include/NetworkRT.h b/include/NetworkRT.h index 0fc9a50..4b6ff73 100644 --- a/include/NetworkRT.h +++ b/include/NetworkRT.h @@ -13,6 +13,7 @@ class NetworkRT { public: nvinfer1::DataType dtRT; nvinfer1::IBuilder *builderRT; + nvinfer1::IRuntime *runtimeRT; nvinfer1::INetworkDefinition *networkRT; nvinfer1::ICudaEngine *engineRT; @@ -32,18 +33,33 @@ public: */ dnnType* infer(dataDim_t &dim, dnnType* data); - nvinfer1::ITensor* convert_layer(nvinfer1::ITensor *input, Layer *l); - nvinfer1::ITensor* convert_layer(nvinfer1::ITensor *input, Conv2d *l); - nvinfer1::ITensor* convert_layer(nvinfer1::ITensor *input, Activation *l); - nvinfer1::ITensor* convert_layer(nvinfer1::ITensor *input, Dense *l); - nvinfer1::ITensor* convert_layer(nvinfer1::ITensor *input, Pooling *l); - nvinfer1::ITensor* convert_layer(nvinfer1::ITensor *input, Softmax *l); - nvinfer1::ITensor* convert_layer(nvinfer1::ITensor *input, Route *l); - nvinfer1::ITensor* convert_layer(nvinfer1::ITensor *input, Reorg *l); - nvinfer1::ITensor* convert_layer(nvinfer1::ITensor *input, Region *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Layer *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Conv2d *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Activation *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Dense *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l); + bool serialize(const char *filename); + bool deserialize(const char *filename); }; +template void writeBUF(char*& buffer, const T& val) +{ + *reinterpret_cast(buffer) = val; + buffer += sizeof(T); +} + +template T readBUF(const char*& buffer) +{ + T val = *reinterpret_cast(buffer); + buffer += sizeof(T); + return val; +} + } #endif //NETWORKRT_H \ No newline at end of file diff --git a/include/utils.h b/include/utils.h index caf8def..922e005 100644 --- a/include/utils.h +++ b/include/utils.h @@ -88,6 +88,7 @@ } void printCenteredTitle(const char *title, char fill, int dim); +bool fileExist(const char *fname); void readBinaryFile(const char* fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0); int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true); void printDeviceVector(int size, dnnType* vec_d, bool device = true); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index c9566c1..9ea82cb 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -35,36 +35,39 @@ NetworkRT::NetworkRT(Network *net) { //add input layer dataDim_t dim = net->layers[0]->input_dim; - ITensor *input = networkRT->addInput("data", dtRT, - DimsCHW{ dim.c, dim.h, dim.w}); - checkNULL(input); + if(!fileExist("net.rt")) { + ITensor *input = networkRT->addInput("data", dtRT, + DimsCHW{ dim.c, dim.h, dim.w}); + checkNULL(input); - //add other layers - for(int i=0; inum_layers; i++) { - Layer *l = net->layers[i]; - input = convert_layer(input, l); - tensors[l] = input; + //add other layers + for(int i=0; inum_layers; i++) { + Layer *l = net->layers[i]; + ILayer *Ilay = convert_layer(input, l); + Ilay->setName( (l->getLayerName() + std::to_string(i)).c_str() ); + + input = Ilay->getOutput(0); + tensors[l] = input; + } + if(input == NULL) + FatalError("conversion failed"); + + //build tensorRT + input->setName("out"); + networkRT->markOutput(*input); + + // Build the engine + builderRT->setMaxBatchSize(1); + builderRT->setMaxWorkspaceSize(1 << 20); + + std::cout<<"Building tensorRT cuda engine...\n"; + engineRT = builderRT->buildCudaEngine(*networkRT); + // we don't need the network any more + //networkRT->destroy(); + serialize("net.rt"); + } else { + deserialize("net.rt"); } - if(input == NULL) - FatalError("conversion failed"); - output_dim = dim; - Dims oDim = input->getDimensions(); - output_dim.c = oDim.d[0]; - output_dim.h = oDim.d[1]; - output_dim.w = oDim.d[2]; - - //build tensorRT - input->setName("out"); - networkRT->markOutput(*input); - - // Build the engine - builderRT->setMaxBatchSize(1); - builderRT->setMaxWorkspaceSize(1 << 20); - - std::cout<<"Building tensorRT cuda engine...\n"; - engineRT = builderRT->buildCudaEngine(*networkRT); - // we don't need the network any more - //networkRT->destroy(); std::cout<<"create execution context\n"; contextRT = engineRT->createExecutionContext(); @@ -73,14 +76,20 @@ NetworkRT::NetworkRT(Network *net) { // of these, but in this case we know that there is exactly one input and one output. if(engineRT->getNbBindings() != 2) FatalError("Incorrect buffers number"); - + // In order to bind the buffers, we need to know the names of the input and output tensors. // note that indices are guaranteed to be less than IEngine::getNbBindings() buf_input_idx = engineRT->getBindingIndex("data"); buf_output_idx = engineRT->getBindingIndex("out"); std::cout<<"input idex = "< output index = "<getBindingDimensions(buf_output_idx); + output_dim.c = oDim.d[0]; + output_dim.h = oDim.d[1]; + output_dim.w = oDim.d[2]; + + // create GPU buffers and a stream checkCuda(cudaMalloc(&buffersRT[buf_input_idx], dim.tot()*sizeof(dnnType))); checkCuda(cudaMalloc(&buffersRT[buf_output_idx], output_dim.tot()*sizeof(dnnType))); checkCuda(cudaMalloc(&output, output_dim.tot()*sizeof(dnnType))); @@ -103,7 +112,7 @@ dnnType* NetworkRT::infer(dataDim_t &dim, dnnType* data) { return output; } -ITensor* NetworkRT::convert_layer(ITensor *input, Layer *l) { +ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) { layerType_t type = l->getLayerType(); @@ -128,7 +137,7 @@ ITensor* NetworkRT::convert_layer(ITensor *input, Layer *l) { return NULL; } -ITensor* NetworkRT::convert_layer(ITensor *input, Dense *l) { +ILayer* NetworkRT::convert_layer(ITensor *input, Dense *l) { //std::cout<<"convert Dense\n"; Weights w { dtRT, l->data_h, l->inputs*l->outputs}; @@ -136,10 +145,10 @@ ITensor* NetworkRT::convert_layer(ITensor *input, Dense *l) { IFullyConnectedLayer *lRT = networkRT->addFullyConnected(*input, l->outputs, w, b); checkNULL(lRT); - return lRT->getOutput(0); + return lRT; } -ITensor* NetworkRT::convert_layer(ITensor *input, Conv2d *l) { +ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) { //std::cout<<"convert conv2D\n"; Weights w { dtRT, l->data_h, l->inputs*l->outputs*l->kernelH*l->kernelW}; @@ -185,13 +194,13 @@ ITensor* NetworkRT::convert_layer(ITensor *input, Conv2d *l) { shift2, scale2, power); checkNULL(lRT3); - return lRT3->getOutput(0); + return lRT3; } - return lRT->getOutput(0); + return lRT; } -ITensor* NetworkRT::convert_layer(ITensor *input, Pooling *l) { +ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) { //std::cout<<"convert Pooling\n"; IPoolingLayer *lRT = networkRT->addPooling(*input, @@ -199,10 +208,10 @@ ITensor* NetworkRT::convert_layer(ITensor *input, Pooling *l) { checkNULL(lRT); lRT->setStride(DimsHW{l->strideH, l->strideW}); - return lRT->getOutput(0); + return lRT; } -ITensor* NetworkRT::convert_layer(ITensor *input, Activation *l) { +ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { //std::cout<<"convert Activation\n"; if(l->act_mode == ACTIVATION_LEAKY) { @@ -210,24 +219,24 @@ ITensor* NetworkRT::convert_layer(ITensor *input, Activation *l) { IPlugin *plugin = new ActivationLeakyRT(); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); - return lRT->getOutput(0); + return lRT; } IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kRELU); checkNULL(lRT); - return lRT->getOutput(0); + return lRT; } -ITensor* NetworkRT::convert_layer(ITensor *input, Softmax *l) { +ILayer* NetworkRT::convert_layer(ITensor *input, Softmax *l) { //std::cout<<"convert softmax\n"; ISoftMaxLayer *lRT = networkRT->addSoftMax(*input); checkNULL(lRT); - return lRT->getOutput(0); + return lRT; } -ITensor* NetworkRT::convert_layer(ITensor *input, Route *l) { +ILayer* NetworkRT::convert_layer(ITensor *input, Route *l) { //std::cout<<"convert route\n"; ITensor *tens[256]; @@ -236,27 +245,85 @@ ITensor* NetworkRT::convert_layer(ITensor *input, Route *l) { IConcatenationLayer *lRT = networkRT->addConcatenation(tens, l->layers_n); checkNULL(lRT); - return lRT->getOutput(0); + return lRT; } -ITensor* NetworkRT::convert_layer(ITensor *input, Reorg *l) { +ILayer* NetworkRT::convert_layer(ITensor *input, Reorg *l) { //std::cout<<"convert Reorg\n"; //std::cout<<"New plugin REORG\n"; IPlugin *plugin = new ReorgRT(l->stride); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); - return lRT->getOutput(0); + return lRT; } -ITensor* NetworkRT::convert_layer(ITensor *input, Region *l) { +ILayer* NetworkRT::convert_layer(ITensor *input, Region *l) { //std::cout<<"convert Region\n"; //std::cout<<"New plugin REGION\n"; IPlugin *plugin = new RegionRT(l->classes, l->coords, l->num, l->thresh); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); - return lRT->getOutput(0); + return lRT; +} + +bool NetworkRT::serialize(const char *filename) { + + std::ofstream p(filename); + if (!p) { + FatalError("could not open plan output file"); + return false; + } + + IHostMemory *ptr = engineRT->serialize(); + if(ptr == nullptr) + FatalError("Cant serialize network"); + + p.write(reinterpret_cast(ptr->data()), ptr->size()); + ptr->destroy(); + return true; +} + +class PluginFactory : IPluginFactory +{ +public: + virtual IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength) { + const char * buf = reinterpret_cast(serialData); + + std::string name(layerName); + + if(name.find("Activation") == 0) { + ActivationLeakyRT *a = new ActivationLeakyRT(); + a->size = readBUF(buf); + return a; + } + FatalError("Cant deserialize Plugin"); + return NULL; + } +}; + +bool NetworkRT::deserialize(const char *filename) { + + char *gieModelStream{nullptr}; + size_t size{0}; + std::ifstream file(filename, std::ios::binary); + if (file.good()) { + file.seekg(0, file.end); + size = file.tellg(); + file.seekg(0, file.beg); + gieModelStream = new char[size]; + file.read(gieModelStream, size); + file.close(); + } + + PluginFactory plfact; + + runtimeRT = createInferRuntime(loggerRT); + engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size, (IPluginFactory *) &plfact); + //if (gieModelStream) delete [] gieModelStream; + + return true; } } diff --git a/src/pluginsRT/ActivationLeakyRT.cpp b/src/pluginsRT/ActivationLeakyRT.cpp index 651eb36..7376857 100644 --- a/src/pluginsRT/ActivationLeakyRT.cpp +++ b/src/pluginsRT/ActivationLeakyRT.cpp @@ -48,10 +48,12 @@ public: virtual size_t getSerializationSize() override { - return 0; + return 1*sizeof(int); } virtual void serialize(void* buffer) override { + char *buf = reinterpret_cast(buffer); + tkDNN::writeBUF(buf, size); } int size; diff --git a/src/utils.cpp b/src/utils.cpp index 8e2a12d..65c040d 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -13,6 +13,11 @@ void printCenteredTitle(const char *title, char fill, int dim) { std::cout.fill(' '); } +bool fileExist(const char *fname) { + std::ifstream dataFile (fname, std::ios::in | std::ios::binary); + return dataFile; +} + void readBinaryFile(const char* fname, int size, dnnType** data_h, dnnType** data_d, int seek) {