Add the Int8 calibrator and the tensorRT Int8 inference

Signed-off-by: Davide Sapienza <sapienza.dav@gmail.com>

include/tkDNN/Int8BatchStream.h

@@ -0,0 +1,65 @@
+#ifndef INT8BATCHSTREAM_H
+#define INT8BATCHSTREAM_H
+
+#include <vector>
+#include <assert.h>
+#include <algorithm>
+#include <iterator>
+#include <stdint.h>
+#include <iostream>
+#include <string>
+#include "NvInfer.h"
+
+#include <fstream>
+#include <iomanip>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/dnn/dnn.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "tkdnn.h"
+#include "utils.h"
+
+class BatchStream
+{
+public:
+	BatchStream(tk::dnn::dataDim_t dim, int batchSize, int maxBatches, const std::string& fileimglist, const std::string& filelabellist);
+	virtual ~BatchStream() {}
+	void reset(int firstBatch);
+	bool next();
+	void skip(int skipCount);
+	float *getBatch() { return mBatch.data();}
+	float *getLabels() { return mLabels.data();}
+	int getBatchesRead() const { return mBatchCount; }
+	int getBatchSize() const { return mBatchSize; }
+	nvinfer1::DimsNCHW getDims() const { return mDims; }
+	float* getFileBatch() { return &mFileBatch[0]; }
+	float* getFileLabels() { return &mFileLabels[0]; }
+	void readInListFile(const std::string& dataFilePath, std::vector<std::string>& mListIn);
+	void readCVimage(std::string inputFileName, std::vector<float>& res, bool fixshape = true);
+	void readLabels(std::string inputFileName ,std::vector<float>& ris);
+	bool update();
+
+private:
+	int mBatchSize{ 0 };
+	int mMaxBatches{ 0 };
+	int mBatchCount{ 0 };
+	int mFileCount{ 0 }, mFileBatchPos{ 0 };
+	int mImageSize{ 0 };
+
+	nvinfer1::DimsNCHW mDims;
+	std::vector<float> mBatch;
+	std::vector<float> mLabels;
+	std::vector<float> mFileBatch;
+	std::vector<float> mFileLabels;
+
+	int mHeight;
+	int mWidth;
+	std::string mFileImgList;
+	std::vector<std::string> mListImg;
+	std::string mFileLabelList;
+	std::vector<std::string> mListLabel;
+};
+
+#endif //INT8BATCHSTREAM

include/tkDNN/Int8Calibrator.h

@@ -0,0 +1,41 @@
+#ifndef INT8CALIBRATOR_H
+#define INT8CALIBRATOR_H
+
+#include <vector>
+#include <assert.h>
+#include <algorithm>
+#include <iterator>
+#include <stdint.h>
+#include <iostream>
+#include <string>
+#include "NvInfer.h"
+
+#include <fstream>
+#include <iomanip>
+
+#include "Int8BatchStream.h"
+
+#include "tkdnn.h"
+#include "utils.h"
+
+class Int8EntropyCalibrator : public nvinfer1::IInt8EntropyCalibrator{
+public:
+	Int8EntropyCalibrator(BatchStream& stream, int firstBatch, const std::string& calibTableFilePath, const std::string& inputBlobName, bool readCache = true);
+	virtual ~Int8EntropyCalibrator() { checkCuda(cudaFree(mDeviceInput)); }
+	int getBatchSize() const override { return mStream.getBatchSize(); }
+	bool getBatch(void* bindings[], const char* names[], int nbBindings) override;
+	const void* readCalibrationCache(size_t& length) override;
+	void writeCalibrationCache(const void* cache, size_t length) override;
+
+private:
+	BatchStream mStream;
+	const std::string mCalibTableFilePath{nullptr};
+	const std::string mInputBlobName;
+	bool mReadCache{ true };
+
+	size_t mInputCount;
+	void* mDeviceInput{ nullptr };
+	std::vector<char> mCalibrationCache;
+};
+
+#endif //INT8CALIBRATOR_H

include/tkDNN/NetworkRT.h

@@ -32,7 +32,6 @@ using namespace nvinfer1;
 #include "pluginsRT/YoloRT.h"
 #include "pluginsRT/UpsampleRT.h"
 #include "pluginsRT/ResizeLayerRT.h"
-//#include "pluginsRT/Int8Calibrator.h"
 #include "pluginsRT/DeformableConvRT.h"
 #include "pluginsRT/FlattenConcatRT.h"
 #include "pluginsRT/ReshapeRT.h"
@@ -56,6 +55,7 @@ public:
     nvinfer1::IBuilder *builderRT;
     nvinfer1::IRuntime *runtimeRT;
     nvinfer1::INetworkDefinition *networkRT; 
+    nvinfer1::IBuilderConfig *configRT;
     
     nvinfer1::ICudaEngine *engineRT;
     nvinfer1::IExecutionContext *contextRT;