From a37c14d71dfca0833b60c211e51eae1010a983b1 Mon Sep 17 00:00:00 2001 From: Adam Cuellar Date: Mon, 11 Apr 2022 15:57:40 -0400 Subject: [PATCH] Fix bad merge --- include/tkDNN/NetworkRT.h | 170 ++++++++------------ include/tkDNN/pluginsRT/ActivationSwishRT.h | 101 +++++++----- src/NetworkRT.cpp | 64 ++------ src/pluginsRT/ActivationSwishRT.cpp | 133 +++++++++++++++ 4 files changed, 273 insertions(+), 195 deletions(-) create mode 100644 src/pluginsRT/ActivationSwishRT.cpp diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index 5e8d980..b607644 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -26,130 +27,89 @@ #include #include -namespace tk { namespace dnn { - -template void writeBUF(char*& buffer, const T& val) -{ - *reinterpret_cast(buffer) = val; - buffer += sizeof(T); -} - -template T readBUF(const char*& buffer) -{ - T val = *reinterpret_cast(buffer); - buffer += sizeof(T); - return val; -} - -using namespace nvinfer1; -#include "pluginsRT/ActivationLeakyRT.h" -#include "pluginsRT/ActivationLogisticRT.h" -#include "pluginsRT/ActivationReLUCeilingRT.h" -#include "pluginsRT/ActivationMishRT.h" -#include "pluginsRT/ActivationSwishRT.h" -#include "pluginsRT/ReorgRT.h" -#include "pluginsRT/RegionRT.h" -#include "pluginsRT/RouteRT.h" -#include "pluginsRT/ShortcutRT.h" -#include "pluginsRT/YoloRT.h" -#include "pluginsRT/UpsampleRT.h" -#include "pluginsRT/ResizeLayerRT.h" -#include "pluginsRT/DeformableConvRT.h" -#include "pluginsRT/FlattenConcatRT.h" -#include "pluginsRT/ReshapeRT.h" -#include "pluginsRT/MaxPoolingFixedSizeRT.h" - -class PluginFactory : IPluginFactory -{ -public: - YoloRT *yolos[16]; - int n_yolos; - - virtual IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength); -}; namespace tk { namespace dnn { -class NetworkRT { + class NetworkRT { -public: - nvinfer1::DataType dtRT; - nvinfer1::IBuilder *builderRT; - nvinfer1::IRuntime *runtimeRT; - nvinfer1::INetworkDefinition *networkRT; -#if NV_TENSORRT_MAJOR >= 6 - nvinfer1::IBuilderConfig *configRT; + public: + nvinfer1::DataType dtRT; + nvinfer1::IBuilder *builderRT; + nvinfer1::IRuntime *runtimeRT; + nvinfer1::INetworkDefinition *networkRT; +#if NV_TENSORRT_MAJOR >= 6 + nvinfer1::IBuilderConfig *configRT; #endif - - nvinfer1::ICudaEngine *engineRT; - nvinfer1::IExecutionContext *contextRT; - const static int MAX_BUFFERS_RT = 10; - void* buffersRT[MAX_BUFFERS_RT]; - dataDim_t buffersDIM[MAX_BUFFERS_RT]; - int buf_input_idx, buf_output_idx; - bool builderActive = false; - dataDim_t input_dim, output_dim; - dnnType *output; - cudaStream_t stream; + nvinfer1::ICudaEngine *engineRT; + nvinfer1::IExecutionContext *contextRT; - std::vector yolo_plugins; // yolo layers in network + const static int MAX_BUFFERS_RT = 10; + void* buffersRT[MAX_BUFFERS_RT]; + dataDim_t buffersDIM[MAX_BUFFERS_RT]; + int buf_input_idx, buf_output_idx; + bool builderActive = false; + dataDim_t input_dim, output_dim; + dnnType *output; + cudaStream_t stream; - NetworkRT(Network *net, const char *name); - virtual ~NetworkRT(); + std::vector yolo_plugins; // yolo layers in network - int getMaxBatchSize() { - if(engineRT != nullptr) - return engineRT->getMaxBatchSize(); - else - return 0; - } + NetworkRT(Network *net, const char *name); + virtual ~NetworkRT(); - int getBuffersN() { - if(engineRT != nullptr) - return engineRT->getNbBindings(); - else - return 0; - } + int getMaxBatchSize() { + if(engineRT != nullptr) + return engineRT->getMaxBatchSize(); + else + return 0; + } - /** - Do inference - */ - dnnType* infer(dataDim_t &dim, dnnType* data); - void enqueue(int batchSize = 1); + int getBuffersN() { + if(engineRT != nullptr) + return engineRT->getNbBindings(); + else + return 0; + } - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Layer *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Conv2d *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Activation *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Dense *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l); - nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Flatten *l); - nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reshape *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Resize *l); - nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reorg *l); - nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Region *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l); - nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Yolo *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, DeformConv2d *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input,Padding *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor* input,MulAdd *l); + /** + Do inference + */ + dnnType* infer(dataDim_t &dim, dnnType* data); + void enqueue(int batchSize = 1); + + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Layer *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Conv2d *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Activation *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Dense *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l); + nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Flatten *l); + nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reshape *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Resize *l); + nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reorg *l); + nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Region *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l); + nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Yolo *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, DeformConv2d *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input,Padding *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor* input,MulAdd *l); #if NV_TENSORRT_MAJOR > 5 && NV_TENSORRT_MAJOR < 8 - bool serialize(const char *filename); + bool serialize(const char *filename); #else - bool serialize(const char *filename,nvinfer1::IHostMemory *ptr); + bool serialize(const char *filename,nvinfer1::IHostMemory *ptr); #endif - bool deserialize(const char *filename); - void destroy(); + bool deserialize(const char *filename); + void destroy(); -}; + }; -}} -#endif //NETWORKRT_H + }} +#endif //NETWORKRT_H \ No newline at end of file diff --git a/include/tkDNN/pluginsRT/ActivationSwishRT.h b/include/tkDNN/pluginsRT/ActivationSwishRT.h index 123ff1f..549e5a0 100644 --- a/include/tkDNN/pluginsRT/ActivationSwishRT.h +++ b/include/tkDNN/pluginsRT/ActivationSwishRT.h @@ -1,61 +1,82 @@ #include #include "../kernels.h" +#include +#include -class ActivationSwishRT : public IPlugin { +namespace nvinfer1 { + class ActivationSwishRT : public IPluginV2 { -public: - ActivationSwishRT() { + public: + ActivationSwishRT() ; + + ~ActivationSwishRT() ; + + ActivationSwishRT(const void *data, size_t length) ; - } + int getNbOutputs() const NOEXCEPT override ; - ~ActivationSwishRT(){ + Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ; - } + void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type, + PluginFormat format, int maxBatchSize) NOEXCEPT override ; - int getNbOutputs() const override { - return 1; - } + int initialize() NOEXCEPT override ; - Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override { - return inputs[0]; - } + void terminate() NOEXCEPT override ; - void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override { - size = 1; - for(int i=0; i 7 + int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,cudaStream_t stream) NOEXCEPT override ; +#elif NV_TENSORRT_MAJOR == 7 + int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override; +#endif - int initialize() override { + size_t getSerializationSize() const NOEXCEPT override ; - return 0; - } + void serialize(void *buffer) const NOEXCEPT override ; - virtual void terminate() override { - } + const char *getPluginType() const NOEXCEPT override ; - virtual size_t getWorkspaceSize(int maxBatchSize) const override { - return 0; - } + const char *getPluginVersion() const NOEXCEPT override ; - virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override { + void destroy() NOEXCEPT override { delete this; } - activationSwishForward((dnnType*)reinterpret_cast(inputs[0]), - reinterpret_cast(outputs[0]), batchSize*size, stream); - return 0; - } + bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ; + const char *getPluginNamespace() const NOEXCEPT override ; - virtual size_t getSerializationSize() override { - return 1*sizeof(int); - } + void setPluginNamespace(const char *plguinNamespace) NOEXCEPT override ; - virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer),*a=buf; - tk::dnn::writeBUF(buf, size); - assert(buf == a + getSerializationSize()); - } + IPluginV2 *clone() const NOEXCEPT override ; - int size; -}; + int size; + private: + std::string mPluginNamespace; + }; + + class ActivationSwishRTPluginCreator : public IPluginCreator { + public: + ActivationSwishRTPluginCreator() ; + + void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ; + const char *getPluginNamespace() const NOEXCEPT override ; + + IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ; + + IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ; + + const char *getPluginName() const NOEXCEPT override ; + + const char *getPluginVersion() const NOEXCEPT override ; + + const PluginFieldCollection *getFieldNames() NOEXCEPT override ; + + private: + static PluginFieldCollection mFC; + static std::vector mPluginAttributes; + std::string mPluginNamespace; + }; + + REGISTER_TENSORRT_PLUGIN(ActivationSwishRTPluginCreator); +}; \ No newline at end of file diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 9617775..9b14970 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -646,17 +646,12 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { checkNULL(lRT); return lRT; } - else if(l->act_mode == CUDNN_ACTIVATION_ELU || l->act_mode == ACTIVATION_ELU){ - IActivationLayer *lRT = networkRT->addActivation(*input,ActivationType::kELU); - else if(l->act_mode == ACTIVATION_SWISH) { - IPlugin *plugin = new ActivationSwishRT(); - IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); - checkNULL(lRT); - return lRT; + else if(l->act_mode == CUDNN_ACTIVATION_ELU || l->act_mode == ACTIVATION_ELU) { + IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kELU); } - else if(l->act_mode == ACTIVATION_LOGISTIC) { - IPlugin *plugin = new ActivationLogisticRT(); - IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); + else if(l->act_mode == ACTIVATION_SWISH) { + IPluginV2 *plugin = new ActivationSwishRT(); + ILayer *lRT = networkRT->addPluginV2(&input, 1, *plugin); checkNULL(lRT); return lRT; } @@ -1034,48 +1029,17 @@ bool NetworkRT::deserialize(const char *filename) { } #if NV_TENSORRT_MAJOR > 7 -void NetworkRT::destroy() { - delete contextRT; - if(builderActive) { - delete engineRT; - delete builderRT; - - -IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialData, size_t serialLength) { - const char * buf = reinterpret_cast(serialData),*bufCheck = buf; - - std::string name(layerName); - //std::cout<(buf)); - a->size = readBUF(buf); - assert(buf == bufCheck + serialLength); - return a; - } - if(name.find("ActivationMish") == 0) { - ActivationMishRT *a = new ActivationMishRT(); - a->size = readBUF(buf); - assert(buf == bufCheck + serialLength); - return a; - } - if(name.find("ActivationSwish") == 0) { - ActivationSwishRT *a = new ActivationSwishRT(); - a->size = readBUF(buf); - assert(buf == bufCheck + serialLength); - return a; - } - if(name.find("ActivationLogistic") == 0) { - ActivationLogisticRT *a = new ActivationLogisticRT(); - a->size = readBUF(buf); - return a; - } -} + void NetworkRT::destroy() { + delete contextRT; + if(builderActive) { + delete engineRT; + delete builderRT; + } + } #elif NV_TENSORRT_MAJOR <=7 -void NetworkRT::destroy() { + void NetworkRT::destroy() { } #endif - -}} +}} \ No newline at end of file diff --git a/src/pluginsRT/ActivationSwishRT.cpp b/src/pluginsRT/ActivationSwishRT.cpp new file mode 100644 index 0000000..3b4b9d0 --- /dev/null +++ b/src/pluginsRT/ActivationSwishRT.cpp @@ -0,0 +1,133 @@ +// +// Created by Adam on 4/11/2022 +// +#include +using namespace nvinfer1; +std::vector ActivationSwishRTPluginCreator::mPluginAttributes; +PluginFieldCollection ActivationSwishRTPluginCreator::mFC{}; + +ActivationSwishRT::ActivationSwishRT() { + +} + +ActivationSwishRT::~ActivationSwishRT() { + +} + +ActivationSwishRT::ActivationSwishRT(const void *data, size_t length) { + const char *buf = reinterpret_cast(data), *bufCheck = buf; + size = readBUF(buf); + assert(buf == bufCheck + length); +} + +int ActivationSwishRT::getNbOutputs() const NOEXCEPT { return 1; } + +Dims ActivationSwishRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT { return inputs[0]; } + +void ActivationSwishRT::configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type, + PluginFormat format, int maxBatchSize) NOEXCEPT { +assert(format == PluginFormat::kLINEAR); +size = 1; +for (int i = 0; i < outputDims[0].nbDims; i++) +size *= outputDims[0].d[i]; +} + +int ActivationSwishRT::initialize() NOEXCEPT { return 0; } + +void ActivationSwishRT::terminate() NOEXCEPT {} + +size_t ActivationSwishRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT { return 0; } + +#if NV_TENSORRT_MAJOR > 7 +int ActivationSwishRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, + cudaStream_t stream) NOEXCEPT { + activationSwishForward((dnnType *) reinterpret_cast(inputs[0]), + reinterpret_cast(outputs[0]), batchSize * size, stream); + return 0; +} +#elif NV_TENSORRT_MAJOR == 7 +int32_t ActivationSwishRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, + cudaStream_t stream) { + activationSwishForward((dnnType *) reinterpret_cast(inputs[0]), + reinterpret_cast(outputs[0]), batchSize * size, stream); + return 0; +} +#endif + +size_t ActivationSwishRT::getSerializationSize() const NOEXCEPT { + return 1 * sizeof(int); +} + +void ActivationSwishRT::serialize(void *buffer) const NOEXCEPT { + char *buf = reinterpret_cast(buffer), *a = buf; + writeBUF(buf, size); + assert(buf == a + getSerializationSize()); +} + +const char* ActivationSwishRT::getPluginType() const NOEXCEPT { + return "ActivationSwishRT_tkDNN"; +} + +const char *ActivationSwishRT::getPluginVersion() const NOEXCEPT { + return "1"; +} + +bool ActivationSwishRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT { + return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR); +} + +const char *ActivationSwishRT::getPluginNamespace() const NOEXCEPT { + return mPluginNamespace.c_str(); +} + +void ActivationSwishRT::setPluginNamespace(const char *plguinNamespace) NOEXCEPT { + mPluginNamespace = plguinNamespace; +} + +IPluginV2 *ActivationSwishRT::clone() const NOEXCEPT { + auto *p = new ActivationSwishRT(); + p->setPluginNamespace(mPluginNamespace.c_str()); + return p; +} + + + +ActivationSwishRTPluginCreator::ActivationSwishRTPluginCreator() { + mPluginAttributes.clear(); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +void ActivationSwishRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT { + mPluginNamespace = pluginNamespace; +} + +const char *ActivationSwishRTPluginCreator::getPluginNamespace() const NOEXCEPT { + return mPluginNamespace.c_str(); +} + +IPluginV2 *ActivationSwishRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT { + auto *pluginObj = new ActivationSwishRT(serialData, serialLength); + pluginObj->setPluginNamespace(mPluginNamespace.c_str()); + return pluginObj; +} + +IPluginV2 *ActivationSwishRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT { + const PluginField *fields = fc->fields; + auto *pluginObj = new ActivationSwishRT(); + pluginObj->setPluginNamespace(mPluginNamespace.c_str()); + return pluginObj; +} + +const char *ActivationSwishRTPluginCreator::getPluginName() const NOEXCEPT { + return "ActivationSwishRT_tkDNN"; +} + +const char *ActivationSwishRTPluginCreator::getPluginVersion() const NOEXCEPT{ + return "1"; +} + +const PluginFieldCollection *ActivationSwishRTPluginCreator::getFieldNames() NOEXCEPT { + return &mFC; +} +