diff --git a/README.md b/README.md index 3d956a5..12bb124 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,36 @@ make during the cmake configuration it will be dowloaded the weights needed for running the tests +## DLA34 and ResNet101 weights +To get weights and outputs needed for running the tests you can use the Python +script and the Anaconda environment included in the repository. + +Create Anaconda environment and activate it: +``` +conda env create -f file_name.yml +source activate env_name +``` +Run the Python script inside the environment. + +## CenterNet weights +To get the weights needed for running the tests: + +* clone the forked repository by the original CenterNet: +``` +git clone https://github.com/sapienzadavide/CenterNet.git +``` +* follow the instruction in the README.md and INSTALL.md +* copy the weigths and outputs from /path/to/CenterNet/src/ in ./test/centernet-path/ . For example: +``` +cp /path/to/CenterNet/src/layers_dla/* ./test/dla34_cnet/layers/ +cp /path/to/CenterNet/src/debug_dla/* ./test/dla34_cnet/debug/ +``` +or +``` +cp /path/to/CenterNet/src/layers_resdcn/* ./test/resnet101_cnet/layers/ +cp /path/to/CenterNet/src/debug_resdcn/* ./test/resnet101_cnet/debug/ +``` + ## Test Assumiung you have correctly builded the library these are the test ready to exec: * test_simple: a simple convolutional and dense network (CUDNN only) @@ -35,6 +65,11 @@ Assumiung you have correctly builded the library these are the test ready to exe * test_yolo: YOLO detection network (CUDNN and TENSORRT) * test_yolo_tiny: smaller version of YOLO (CUDNN and TENSRRT) * test_yolo3_berkeley: our yolo3 version trained with BDD100K dateset +* test_resnet101: ResNet101 network (CUDNN and TENSORRT) +* test_resnet101_cnet: CenterNet detection based on ResNet101 (CUDNN and TENSORRT) +* test_dla34: DLA34 network (CUDNN and TENSORRT) +* test_dla34_cnet: CenterNet detection based on DLA34 (CUDNN and TENSORRT) + ## yolo3 berkeley demo detection For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process: @@ -50,3 +85,31 @@ this will genereate a yolo3_berkeley.rt file that can be used for live detection ./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0 ``` ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) + + +## CenterNet (DLA34, ResNet101) demo detection +For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process: +``` +export TKDNN_MODE=FP16 # set the half floating point optimization +``` + +For CenterNet based on ResNet101: +``` +rm resnet101_cnet.rt # be sure to delete(or move) old tensorRT files +./test_resnet101_cnet # run the yolo test (is slow) +# with f16 inference the result will be a bit incorrect +``` + +For CenterNet based on DLA34: +``` +rm dla34_cnet.rt # be sure to delete(or move) old tensorRT files +./test_dla34_cnet # run the yolo test (is slow) +# with f16 inference the result will be a bit incorrect +``` + +this will genereate resnet101_cnet.rt and dla34_cnet.rt file that can be used for live detection: +``` +./centernet_demo # launch detection on a demo video +./centernet_demo resnet101_cnet.rt /dev/video0 # launch detection on device 0 +./centernet_demo dla34_cnet.rt /dev/video0 # launch detection on device 0 +``` \ No newline at end of file diff --git a/tests/dla34/dla34_weightsexporter.py b/tests/dla34/dla34_weightsexporter.py new file mode 100644 index 0000000..4a412df --- /dev/null +++ b/tests/dla34/dla34_weightsexporter.py @@ -0,0 +1,162 @@ +import torch +import urllib +from PIL import Image +from torchvision import transforms +import numpy as np +import struct +import os + +from pytorchcv.model_provider import get_model as ptcv_get_model +from torch.autograd import Variable + +from torchsummary import summary +import torch.nn as nn + +from torch.jit import trace + +def create_folders(): + if not os.path.exists('debug'): + os.makedirs('debug') + if not os.path.exists('layers'): + os.makedirs('layers') + +def bin_write(f, data): + data =data.flatten() + fmt = 'f'*len(data) + bin = struct.pack(fmt, *data) + f.write(bin) + +def hook(module, input, output): + setattr(module, "_value_hook", output) + +def load_ex_image(model): + # Download an example image from the pytorch website + url, filename = ( + "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg") + try: + urllib.URLopener().retrieve(url, filename) + except: + urllib.request.urlretrieve(url, filename) + + # sample execution (requires torchvision) + input_image = Image.open(filename) + print("input_image: ",input_image.size) + preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[ + 0.229, 0.224, 0.225]), + ]) + input_tensor = preprocess(input_image) + print("input_tensor: ",input_tensor.shape) + # create a mini-batch as expected by the model + input_batch = input_tensor.unsqueeze(0) + + # move the input and model to GPU for speed if available + if torch.cuda.is_available(): + input_batch = input_batch.to('cuda') + model.to('cuda') + + return model, input_batch + +def exp_input(model, input_batch): + # Export the input batch + model(input_batch) + i = input_batch.cpu().data.numpy() + i = np.array(i, dtype=np.float32) + i.tofile("debug/input.bin", format="f") + print("input: ", i.shape) + +def print_wb_output(model): + f = None + for n, m in model.named_modules(): + m.eval() + if 'DLAResBlock' in str(m.type): + continue + + in_output = m._value_hook + o = in_output.data.numpy() + o = np.array(o, dtype=np.float32) + + t = '-'.join(n.split('.')) + o.tofile("debug/" + t + ".bin", format="f") + print('------- ', n, ' ------') + print("debug ",o.shape) + + if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)): + continue + + if ' of Conv2d' in str(m.type) or ' of Linear' in str(m.type): + file_name = "layers/" + t + ".bin" + print("open file: ", file_name) + f = open(file_name, mode='wb') + + w = np.array([]) + b = np.array([]) + if 'weight' in m._parameters and m._parameters['weight'] is not None: + w = m._parameters['weight'].data.numpy() + w = np.array(w, dtype=np.float32) + print (" weights shape:", np.shape(w)) + + if 'bias' in m._parameters and m._parameters['bias'] is not None: + b = m._parameters['bias'].data.numpy() + b = np.array(b, dtype=np.float32) + print (" bias shape:", np.shape(b)) + + if 'BatchNorm2d' in str(m.type): + b = m._parameters['bias'].data.numpy() + b = np.array(b, dtype=np.float32) + s = m._parameters['weight'].data.numpy() + s = np.array(s, dtype=np.float32) + rm = m.running_mean.data.numpy() + rm = np.array(rm, dtype=np.float32) + rv = m.running_var.data.numpy() + rv = np.array(rv, dtype=np.float32) + bin_write(f,b) + bin_write(f,s) + bin_write(f,rm) + bin_write(f,rv) + print (" b shape:", np.shape(b)) + print (" s shape:", np.shape(s)) + print (" rm shape:", np.shape(rm)) + print (" rv shape:", np.shape(rv)) + + else: + bin_write(f,w) + if b.size > 0 and b is not None: + bin_write(f,b) + + if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type): + f.close() + print("close file") + f = None + +if __name__ == '__main__': + model = ptcv_get_model("dla34", pretrained=True) + model.eval() + + # load an example image and load it on model + model, input_batch = load_ex_image(model) + model.eval() + with torch.no_grad(): + output = model(input_batch) + + # create folders debug and layers if do not exist + create_folders() + + # add output attribute to the layers + for n, m in model.named_modules(): + m.register_forward_hook(hook) + + # export input bin + exp_input(model, input_batch) + + print_wb_output(model) + + with open("dla34.txt", 'w') as f: + for item in list(model.children()): + f.write("%s\n" % item) + + summary(model, (3, 224, 224)) + # print(trace(model, input_batch)) diff --git a/tests/dla34/env_dla34.yml b/tests/dla34/env_dla34.yml new file mode 100644 index 0000000..4e27db2 --- /dev/null +++ b/tests/dla34/env_dla34.yml @@ -0,0 +1,60 @@ +name: dla34 +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _pytorch_select=0.2=gpu_0 + - blas=1.0=mkl + - ca-certificates=2019.10.16=0 + - certifi=2019.9.11=py36_0 + - cffi=1.13.1=py36h2e261b9_0 + - cudatoolkit=10.0.130=0 + - cudnn=7.6.0=cuda10.0_0 + - freetype=2.9.1=h8a8886c_1 + - intel-openmp=2019.4=243 + - jpeg=9b=h024ee3a_2 + - libedit=3.1.20181209=hc058e9b_0 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=9.1.0=hdf63c60_0 + - libgfortran-ng=7.3.0=hdf63c60_0 + - libpng=1.6.37=hbc83047_0 + - libstdcxx-ng=9.1.0=hdf63c60_0 + - libtiff=4.0.10=h2733197_2 + - mkl=2019.4=243 + - mkl-service=2.3.0=py36he904b0f_0 + - mkl_fft=1.0.14=py36ha843d7b_0 + - mkl_random=1.1.0=py36hd6b4f25_0 + - ncurses=6.1=he6710b0_1 + - ninja=1.9.0=py36hfd86e86_0 + - numpy=1.17.2=py36haad9e8e_0 + - numpy-base=1.17.2=py36hde5b4d6_0 + - olefile=0.46=py36_0 + - openssl=1.1.1d=h7b6447c_3 + - pillow=6.2.0=py36h34e0f95_0 + - pip=19.3.1=py36_0 + - pycparser=2.19=py36_0 + - python=3.6.9=h265db76_0 + - readline=7.0=h7b6447c_5 + - setuptools=41.6.0=py36_0 + - six=1.12.0=py36_0 + - sqlite=3.30.1=h7b6447c_0 + - tk=8.6.8=hbc83047_0 + - wheel=0.33.6=py36_0 + - xz=5.2.4=h14c3975_4 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.3.7=h0b5b093_0 + - pip: + - chardet==3.0.4 + - decorator==4.4.1 + - idna==2.8 + - lxml==4.4.2 + - networkx==2.4 + - nltk==3.4.5 + - pytorchcv==0.0.55 + - requests==2.22.0 + - summary==0.2.0 + - torch==1.3.0 + - torchsummary==1.5.1 + - torchvision==0.4.1 + - urllib3==1.25.8 + diff --git a/tests/resnet101/env_resnet101.yml b/tests/resnet101/env_resnet101.yml new file mode 100644 index 0000000..35987c3 --- /dev/null +++ b/tests/resnet101/env_resnet101.yml @@ -0,0 +1,56 @@ +name: resnet101 +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _pytorch_select=0.2=gpu_0 + - blas=1.0=mkl + - ca-certificates=2019.10.16=0 + - certifi=2019.9.11=py36_0 + - cffi=1.13.1=py36h2e261b9_0 + - cudatoolkit=10.0.130=0 + - cudnn=7.6.0=cuda10.0_0 + - freetype=2.9.1=h8a8886c_1 + - intel-openmp=2019.4=243 + - jpeg=9b=h024ee3a_2 + - libedit=3.1.20181209=hc058e9b_0 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=9.1.0=hdf63c60_0 + - libgfortran-ng=7.3.0=hdf63c60_0 + - libpng=1.6.37=hbc83047_0 + - libstdcxx-ng=9.1.0=hdf63c60_0 + - libtiff=4.0.10=h2733197_2 + - mkl=2019.4=243 + - mkl-service=2.3.0=py36he904b0f_0 + - mkl_fft=1.0.14=py36ha843d7b_0 + - mkl_random=1.1.0=py36hd6b4f25_0 + - ncurses=6.1=he6710b0_1 + - ninja=1.9.0=py36hfd86e86_0 + - numpy=1.17.2=py36haad9e8e_0 + - numpy-base=1.17.2=py36hde5b4d6_0 + - olefile=0.46=py36_0 + - openssl=1.1.1d=h7b6447c_3 + - pillow=6.2.0=py36h34e0f95_0 + - pip=19.3.1=py36_0 + - pycparser=2.19=py36_0 + - python=3.6.9=h265db76_0 + - pytorch=1.2.0=cuda100py36h938c94c_0 + - readline=7.0=h7b6447c_5 + - setuptools=41.6.0=py36_0 + - six=1.12.0=py36_0 + - sqlite=3.30.1=h7b6447c_0 + - tk=8.6.8=hbc83047_0 + - wheel=0.33.6=py36_0 + - xz=5.2.4=h14c3975_4 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.3.7=h0b5b093_0 + - pip: + - chardet==3.0.4 + - idna==2.8 + - pytorchcv==0.0.55 + - requests==2.22.0 + - torch==1.3.0 + - torchsummary==1.5.1 + - torchvision==0.4.1 + - urllib3==1.25.8 + diff --git a/tests/resnet101/resnet101_weightsexporter.py b/tests/resnet101/resnet101_weightsexporter.py index 140154e..e10a037 100644 --- a/tests/resnet101/resnet101_weightsexporter.py +++ b/tests/resnet101/resnet101_weightsexporter.py @@ -2,15 +2,26 @@ import torch import urllib from PIL import Image from torchvision import transforms -from torchsummary import summary import numpy as np import struct +import os +from pytorchcv.model_provider import get_model as ptcv_get_model +from torch.autograd import Variable + +from torchsummary import summary import torch.nn as nn +from torch.jit import trace + +def create_folders(): + if not os.path.exists('debug'): + os.makedirs('debug') + if not os.path.exists('layers'): + os.makedirs('layers') + def bin_write(f, data): data =data.flatten() - # print(data) fmt = 'f'*len(data) bin = struct.pack(fmt, *data) f.write(bin) @@ -18,38 +29,46 @@ def bin_write(f, data): def hook(module, input, output): setattr(module, "_value_hook", output) +def load_ex_image(model): + # Download an example image from the pytorch website + url, filename = ( + "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg") + try: + urllib.URLopener().retrieve(url, filename) + except: + urllib.request.urlretrieve(url, filename) + + # sample execution (requires torchvision) + input_image = Image.open(filename) + print("input_image: ",input_image.size) + preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[ + 0.229, 0.224, 0.225]), + ]) + input_tensor = preprocess(input_image) + print("input_tensor: ",input_tensor.shape) + # create a mini-batch as expected by the model + input_batch = input_tensor.unsqueeze(0) + # move the input and model to GPU for speed if available + if torch.cuda.is_available(): + input_batch = input_batch.to('cuda') + model.to('cuda') + + return model, input_batch -def print_wb(model, folder): - for name, param in model.named_parameters(): - # print ("Layer", name) - t = name.split('.')[0:-1] - arg = name.split('.')[-1] - t = '-'.join(t) - print (" type: ", t) - - if arg == 'weight': - w = param.data.numpy() - print (" weights shape:", np.shape(w)) - w.tofile(folder + "/" + t + ".bin", format="f") - elif arg == 'bias': - b = param.data.numpy() - print (" bias shape:", np.shape(b)) - b.tofile(folder + "/" + t + ".bias.bin", format="f") - else: - print("Ops!") - - -def print_wb_output(model, input_batch): - for n, m in model.named_modules(): - m.register_forward_hook(hook) - +def exp_input(model, input_batch): + # Export the input batch model(input_batch) - i = input_batch.data.numpy() + i = input_batch.cpu().data.numpy() i = np.array(i, dtype=np.float32) - print(i.shape) i.tofile("debug/input.bin", format="f") + print("input: ", i.shape) +def print_wb_output(model): f = None for n, m in model.named_modules(): in_output = m._value_hook @@ -57,7 +76,9 @@ def print_wb_output(model, input_batch): o = np.array(o, dtype=np.float32) t = '-'.join(n.split('.')) o.tofile("debug/" + t + ".bin", format="f") - + print('------- ', n, ' ------') + print("debug ",o.shape) + if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)): continue @@ -66,14 +87,8 @@ def print_wb_output(model, input_batch): print("open file: ", file_name) f = open(file_name, mode='wb') - print(n, ' ----------------------------------------------------------------') - # print(m._parameters) - #print(m.type) - w = np.array([]) b = np.array([]) - - if 'weight' in m._parameters and m._parameters['weight'] is not None: w = m._parameters['weight'].data.numpy() w = np.array(w, dtype=np.float32) @@ -83,9 +98,6 @@ def print_wb_output(model, input_batch): b = m._parameters['bias'].data.numpy() b = np.array(b, dtype=np.float32) print (" bias shape:", np.shape(b)) - # else: - # b = np.zeros(w.shape[0], dtype=np.float32) - # print (" bias shape:", np.shape(b)) if 'BatchNorm2d' in str(m.type): b = m._parameters['bias'].data.numpy() @@ -96,30 +108,24 @@ def print_wb_output(model, input_batch): rm = np.array(rm, dtype=np.float32) rv = m.running_var.data.numpy() rv = np.array(rv, dtype=np.float32) - #s.tofile(f, format="f") bin_write(f,b) bin_write(f,s) bin_write(f,rm) bin_write(f,rv) + print (" b shape:", np.shape(b)) print (" s shape:", np.shape(s)) print (" rm shape:", np.shape(rm)) print (" rv shape:", np.shape(rv)) else: - - # w.tofile(f, format="f") bin_write(f,w) - # print("w- ",w) if b.size > 0: - # b.tofile(f, format="f") bin_write(f,b) - # print("b - ",b) if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type): f.close() print("close file") f = None - # return @@ -130,34 +136,27 @@ if __name__ == '__main__': model = torch.hub.load('pytorch/vision', 'resnet101', pretrained=True) model.eval() - # Download an example image from the pytorch website - url, filename = ("https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg") - try: urllib.URLopener().retrieve(url, filename) - except: urllib.request.urlretrieve(url, filename) - - # sample execution (requires torchvision) - input_image = Image.open(filename) - preprocess = transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), - ]) - input_tensor = preprocess(input_image) - input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model - - # move the input and model to GPU for speed if available - if torch.cuda.is_available(): - input_batch = input_batch.to('cuda') - model.to('cuda') - + # load an example image and load it on model + model, input_batch = load_ex_image(model) + model.eval() with torch.no_grad(): output = model(input_batch) - # Tensor of shape 1000, with confidence scores over Imagenet's 1000 classes - # print(output) + # create folders debug and layers if do not exist + create_folders() + # add output attribute to the layers + for n, m in model.named_modules(): + m.register_forward_hook(hook) - print_wb_output(model, input_batch) + # export input bin + exp_input(model, input_batch) - # print(list(model.children())) + print_wb_output(model) + + with open("resnet101.txt", 'w') as f: + for item in list(model.children()): + f.write("%s\n" % item) + + summary(model, (3, 224, 224)) + # print(trace(model, input_batch))