Merge branch 'cnet' of https://github.com/ceccocats/tkDNN into cnet

README.md

@@ -27,6 +27,36 @@ make
 during the cmake configuration it will be dowloaded the weights needed for running
 the tests
 
+## DLA34 and ResNet101 weights
+To get weights and outputs needed for running the tests you can use the Python 
+script and the Anaconda environment included in the repository.   
+
+Create Anaconda environment and activate it:
+```
+conda env create -f file_name.yml
+source activate env_name 
+```
+Run the Python script inside the environment.
+
+## CenterNet weights
+To get the weights needed for running the tests:
+
+* clone the forked repository by the original CenterNet:
+```
+git clone https://github.com/sapienzadavide/CenterNet.git
+```
+* follow the instruction in the README.md and INSTALL.md
+* copy the weigths and outputs from /path/to/CenterNet/src/ in ./test/centernet-path/ . For example:
+```
+cp /path/to/CenterNet/src/layers_dla/* ./test/dla34_cnet/layers/
+cp /path/to/CenterNet/src/debug_dla/* ./test/dla34_cnet/debug/
+```
+or
+```
+cp /path/to/CenterNet/src/layers_resdcn/* ./test/resnet101_cnet/layers/
+cp /path/to/CenterNet/src/debug_resdcn/* ./test/resnet101_cnet/debug/
+```
+
 ## Test
 Assumiung you have correctly builded the library these are the test ready to exec:
 * test_simple: a simple convolutional and dense network (CUDNN only)
@@ -35,6 +65,11 @@ Assumiung you have correctly builded the library these are the test ready to exe
 * test_yolo: YOLO detection network (CUDNN and TENSORRT)
 * test_yolo_tiny: smaller version of YOLO (CUDNN and TENSRRT)
 * test_yolo3_berkeley: our yolo3 version trained with BDD100K dateset 
+* test_resnet101: ResNet101 network (CUDNN and TENSORRT)
+* test_resnet101_cnet: CenterNet detection based on ResNet101 (CUDNN and TENSORRT)
+* test_dla34: DLA34 network (CUDNN and TENSORRT)
+* test_dla34_cnet: CenterNet detection based on DLA34 (CUDNN and TENSORRT)
+
 
 ## yolo3 berkeley demo detection
 For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process:
@@ -50,3 +85,31 @@ this will genereate a yolo3_berkeley.rt file that can be used for live detection
 ./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0
 ```
 ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif)
+
+
+## CenterNet (DLA34, ResNet101) demo detection
+For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process:
+```
+export TKDNN_MODE=FP16   # set the half floating point optimization
+```
+
+For CenterNet based on ResNet101:
+```
+rm resnet101_cnet.rt		 # be sure to delete(or move) old tensorRT files
+./test_resnet101_cnet              # run the yolo test (is slow)
+# with f16 inference the result will be a bit incorrect
+```
+
+For CenterNet based on DLA34:
+```
+rm dla34_cnet.rt		     # be sure to delete(or move) old tensorRT files
+./test_dla34_cnet                  # run the yolo test (is slow)
+# with f16 inference the result will be a bit incorrect
+```
+
+this will genereate resnet101_cnet.rt and dla34_cnet.rt file that can be used for live detection:
+```
+./centernet_demo                               # launch detection on a demo video
+./centernet_demo resnet101_cnet.rt /dev/video0 # launch detection on device 0
+./centernet_demo dla34_cnet.rt /dev/video0     # launch detection on device 0
+```

tests/dla34/dla34_weightsexporter.py

@@ -0,0 +1,162 @@
+import torch
+import urllib
+from PIL import Image
+from torchvision import transforms
+import numpy as np
+import struct 
+import os
+
+from pytorchcv.model_provider import get_model as ptcv_get_model
+from torch.autograd import Variable
+
+from torchsummary import summary
+import torch.nn as nn
+
+from torch.jit import trace
+
+def create_folders():
+    if not os.path.exists('debug'):
+        os.makedirs('debug')
+    if not os.path.exists('layers'):
+        os.makedirs('layers')
+
+def bin_write(f, data):
+    data =data.flatten()
+    fmt = 'f'*len(data)
+    bin = struct.pack(fmt, *data)
+    f.write(bin)
+
+def hook(module, input, output):
+    setattr(module, "_value_hook", output)
+
+def load_ex_image(model):
+    # Download an example image from the pytorch website
+    url, filename = (
+        "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg")
+    try:
+        urllib.URLopener().retrieve(url, filename)
+    except:
+        urllib.request.urlretrieve(url, filename)
+    
+    # sample execution (requires torchvision)
+    input_image = Image.open(filename)
+    print("input_image: ",input_image.size)
+    preprocess = transforms.Compose([
+        transforms.Resize(256),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[
+                             0.229, 0.224, 0.225]),
+    ])
+    input_tensor = preprocess(input_image)
+    print("input_tensor: ",input_tensor.shape)
+    # create a mini-batch as expected by the model
+    input_batch = input_tensor.unsqueeze(0)
+
+    # move the input and model to GPU for speed if available
+    if torch.cuda.is_available():
+        input_batch = input_batch.to('cuda')
+        model.to('cuda')
+    
+    return model, input_batch
+
+def exp_input(model, input_batch):
+    # Export the input batch 
+    model(input_batch)
+    i = input_batch.cpu().data.numpy()
+    i = np.array(i, dtype=np.float32)
+    i.tofile("debug/input.bin", format="f")
+    print("input: ", i.shape)
+
+def print_wb_output(model):
+    f = None
+    for n, m in model.named_modules():
+        m.eval()
+        if 'DLAResBlock' in str(m.type):
+            continue
+
+        in_output = m._value_hook
+        o = in_output.data.numpy()
+        o = np.array(o, dtype=np.float32)
+        
+        t = '-'.join(n.split('.'))
+        o.tofile("debug/" + t + ".bin", format="f")
+        print('------- ', n, ' ------') 
+        print("debug  ",o.shape)
+
+        if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)):
+            continue
+        
+        if ' of Conv2d' in str(m.type) or ' of Linear' in str(m.type):
+            file_name = "layers/" + t + ".bin"
+            print("open file: ", file_name)
+            f = open(file_name, mode='wb')
+
+        w = np.array([])
+        b = np.array([])
+        if 'weight' in m._parameters and m._parameters['weight'] is not None:
+            w = m._parameters['weight'].data.numpy()
+            w = np.array(w, dtype=np.float32)
+            print ("    weights shape:", np.shape(w))
+        
+        if 'bias' in m._parameters and m._parameters['bias'] is not None:
+            b = m._parameters['bias'].data.numpy()
+            b = np.array(b, dtype=np.float32)
+            print ("    bias shape:", np.shape(b))
+        
+        if 'BatchNorm2d' in str(m.type):
+            b = m._parameters['bias'].data.numpy()
+            b = np.array(b, dtype=np.float32)
+            s = m._parameters['weight'].data.numpy()
+            s = np.array(s, dtype=np.float32)
+            rm = m.running_mean.data.numpy()
+            rm = np.array(rm, dtype=np.float32)
+            rv = m.running_var.data.numpy()
+            rv = np.array(rv, dtype=np.float32)
+            bin_write(f,b)
+            bin_write(f,s)
+            bin_write(f,rm)
+            bin_write(f,rv)
+            print ("    b shape:", np.shape(b))
+            print ("    s shape:", np.shape(s))
+            print ("    rm shape:", np.shape(rm))
+            print ("    rv shape:", np.shape(rv))
+            
+        else:
+            bin_write(f,w)
+            if b.size > 0 and b is not None:
+                bin_write(f,b)
+                
+        if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type):
+            f.close()
+            print("close file")
+            f = None
+
+if __name__ == '__main__':
+    model = ptcv_get_model("dla34", pretrained=True)
+    model.eval()
+
+    # load an example image and load it on model
+    model, input_batch = load_ex_image(model)
+    model.eval()
+    with torch.no_grad():
+        output = model(input_batch)
+
+    # create folders debug and layers if do not exist
+    create_folders()
+
+    # add output attribute to the layers
+    for n, m in model.named_modules():
+        m.register_forward_hook(hook)
+
+    # export input bin
+    exp_input(model, input_batch)
+
+    print_wb_output(model)
+
+    with open("dla34.txt", 'w') as f:
+        for item in list(model.children()):
+            f.write("%s\n" % item)
+
+    summary(model, (3, 224, 224))
+    # print(trace(model, input_batch))

tests/dla34/env_dla34.yml

@@ -0,0 +1,60 @@
+name: dla34
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _pytorch_select=0.2=gpu_0
+  - blas=1.0=mkl
+  - ca-certificates=2019.10.16=0
+  - certifi=2019.9.11=py36_0
+  - cffi=1.13.1=py36h2e261b9_0
+  - cudatoolkit=10.0.130=0
+  - cudnn=7.6.0=cuda10.0_0
+  - freetype=2.9.1=h8a8886c_1
+  - intel-openmp=2019.4=243
+  - jpeg=9b=h024ee3a_2
+  - libedit=3.1.20181209=hc058e9b_0
+  - libffi=3.2.1=hd88cf55_4
+  - libgcc-ng=9.1.0=hdf63c60_0
+  - libgfortran-ng=7.3.0=hdf63c60_0
+  - libpng=1.6.37=hbc83047_0
+  - libstdcxx-ng=9.1.0=hdf63c60_0
+  - libtiff=4.0.10=h2733197_2
+  - mkl=2019.4=243
+  - mkl-service=2.3.0=py36he904b0f_0
+  - mkl_fft=1.0.14=py36ha843d7b_0
+  - mkl_random=1.1.0=py36hd6b4f25_0
+  - ncurses=6.1=he6710b0_1
+  - ninja=1.9.0=py36hfd86e86_0
+  - numpy=1.17.2=py36haad9e8e_0
+  - numpy-base=1.17.2=py36hde5b4d6_0
+  - olefile=0.46=py36_0
+  - openssl=1.1.1d=h7b6447c_3
+  - pillow=6.2.0=py36h34e0f95_0
+  - pip=19.3.1=py36_0
+  - pycparser=2.19=py36_0
+  - python=3.6.9=h265db76_0
+  - readline=7.0=h7b6447c_5
+  - setuptools=41.6.0=py36_0
+  - six=1.12.0=py36_0
+  - sqlite=3.30.1=h7b6447c_0
+  - tk=8.6.8=hbc83047_0
+  - wheel=0.33.6=py36_0
+  - xz=5.2.4=h14c3975_4
+  - zlib=1.2.11=h7b6447c_3
+  - zstd=1.3.7=h0b5b093_0
+  - pip:
+    - chardet==3.0.4
+    - decorator==4.4.1
+    - idna==2.8
+    - lxml==4.4.2
+    - networkx==2.4
+    - nltk==3.4.5
+    - pytorchcv==0.0.55
+    - requests==2.22.0
+    - summary==0.2.0
+    - torch==1.3.0
+    - torchsummary==1.5.1
+    - torchvision==0.4.1
+    - urllib3==1.25.8
+

tests/resnet101/env_resnet101.yml

@@ -0,0 +1,56 @@
+name: resnet101
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _pytorch_select=0.2=gpu_0
+  - blas=1.0=mkl
+  - ca-certificates=2019.10.16=0
+  - certifi=2019.9.11=py36_0
+  - cffi=1.13.1=py36h2e261b9_0
+  - cudatoolkit=10.0.130=0
+  - cudnn=7.6.0=cuda10.0_0
+  - freetype=2.9.1=h8a8886c_1
+  - intel-openmp=2019.4=243
+  - jpeg=9b=h024ee3a_2
+  - libedit=3.1.20181209=hc058e9b_0
+  - libffi=3.2.1=hd88cf55_4
+  - libgcc-ng=9.1.0=hdf63c60_0
+  - libgfortran-ng=7.3.0=hdf63c60_0
+  - libpng=1.6.37=hbc83047_0
+  - libstdcxx-ng=9.1.0=hdf63c60_0
+  - libtiff=4.0.10=h2733197_2
+  - mkl=2019.4=243
+  - mkl-service=2.3.0=py36he904b0f_0
+  - mkl_fft=1.0.14=py36ha843d7b_0
+  - mkl_random=1.1.0=py36hd6b4f25_0
+  - ncurses=6.1=he6710b0_1
+  - ninja=1.9.0=py36hfd86e86_0
+  - numpy=1.17.2=py36haad9e8e_0
+  - numpy-base=1.17.2=py36hde5b4d6_0
+  - olefile=0.46=py36_0
+  - openssl=1.1.1d=h7b6447c_3
+  - pillow=6.2.0=py36h34e0f95_0
+  - pip=19.3.1=py36_0
+  - pycparser=2.19=py36_0
+  - python=3.6.9=h265db76_0
+  - pytorch=1.2.0=cuda100py36h938c94c_0
+  - readline=7.0=h7b6447c_5
+  - setuptools=41.6.0=py36_0
+  - six=1.12.0=py36_0
+  - sqlite=3.30.1=h7b6447c_0
+  - tk=8.6.8=hbc83047_0
+  - wheel=0.33.6=py36_0
+  - xz=5.2.4=h14c3975_4
+  - zlib=1.2.11=h7b6447c_3
+  - zstd=1.3.7=h0b5b093_0
+  - pip:
+    - chardet==3.0.4
+    - idna==2.8
+    - pytorchcv==0.0.55
+    - requests==2.22.0
+    - torch==1.3.0
+    - torchsummary==1.5.1
+    - torchvision==0.4.1
+    - urllib3==1.25.8
+

tests/resnet101/resnet101_weightsexporter.py

@@ -2,15 +2,26 @@ import torch
 import urllib
 from PIL import Image
 from torchvision import transforms
-from torchsummary import summary
 import numpy as np
 import struct 
+import os
 
+from pytorchcv.model_provider import get_model as ptcv_get_model
+from torch.autograd import Variable
+
+from torchsummary import summary
 import torch.nn as nn
 
+from torch.jit import trace
+
+def create_folders():
+    if not os.path.exists('debug'):
+        os.makedirs('debug')
+    if not os.path.exists('layers'):
+        os.makedirs('layers')
+
 def bin_write(f, data):
     data =data.flatten()
-    # print(data)
     fmt = 'f'*len(data)
     bin = struct.pack(fmt, *data)
     f.write(bin)
@@ -18,38 +29,46 @@ def bin_write(f, data):
 def hook(module, input, output):
     setattr(module, "_value_hook", output)
 
+def load_ex_image(model):
+    # Download an example image from the pytorch website
+    url, filename = (
+        "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg")
+    try:
+        urllib.URLopener().retrieve(url, filename)
+    except:
+        urllib.request.urlretrieve(url, filename)
     
+    # sample execution (requires torchvision)
+    input_image = Image.open(filename)
+    print("input_image: ",input_image.size)
+    preprocess = transforms.Compose([
+        transforms.Resize(256),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[
+                             0.229, 0.224, 0.225]),
+    ])
+    input_tensor = preprocess(input_image)
+    print("input_tensor: ",input_tensor.shape)
+    # create a mini-batch as expected by the model
+    input_batch = input_tensor.unsqueeze(0)
 
-def print_wb(model, folder):
+    # move the input and model to GPU for speed if available
-    for name, param in model.named_parameters():
+    if torch.cuda.is_available():
-        # print ("Layer", name)
+        input_batch = input_batch.to('cuda')
-        t = name.split('.')[0:-1]
+        model.to('cuda')
-        arg = name.split('.')[-1]
-        t = '-'.join(t)
-        print ("    type: ", t)
     
-        if arg == 'weight':
+    return model, input_batch
-            w = param.data.numpy()
-            print ("    weights shape:", np.shape(w))
-            w.tofile(folder + "/" + t + ".bin", format="f")
-        elif arg == 'bias':
-            b = param.data.numpy()
-            print ("    bias shape:", np.shape(b))
-            b.tofile(folder + "/" + t + ".bias.bin", format="f")
-        else:
-            print("Ops!")
-
-
-def print_wb_output(model, input_batch):
-    for n, m in model.named_modules():
-        m.register_forward_hook(hook)
 
+def exp_input(model, input_batch):
+    # Export the input batch 
     model(input_batch)
-    i = input_batch.data.numpy()
+    i = input_batch.cpu().data.numpy()
     i = np.array(i, dtype=np.float32)
-    print(i.shape)
     i.tofile("debug/input.bin", format="f")
+    print("input: ", i.shape)
 
+def print_wb_output(model):
     f = None
     for n, m in model.named_modules():
         in_output = m._value_hook
@@ -57,6 +76,8 @@ def print_wb_output(model, input_batch):
         o = np.array(o, dtype=np.float32)
         t = '-'.join(n.split('.'))
         o.tofile("debug/" + t + ".bin", format="f")
+        print('------- ', n, ' ------') 
+        print("debug  ",o.shape)
         
         if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)):
             continue
@@ -66,14 +87,8 @@ def print_wb_output(model, input_batch):
             print("open file: ", file_name)
             f = open(file_name, mode='wb')
 
-        print(n, ' ----------------------------------------------------------------')
-        # print(m._parameters)
-        #print(m.type)
-
         w = np.array([])
         b = np.array([])
-
-
         if 'weight' in m._parameters and m._parameters['weight'] is not None:
             w = m._parameters['weight'].data.numpy()
             w = np.array(w, dtype=np.float32)
@@ -83,9 +98,6 @@ def print_wb_output(model, input_batch):
             b = m._parameters['bias'].data.numpy()
             b = np.array(b, dtype=np.float32)
             print ("    bias shape:", np.shape(b))
-        # else:
-        #     b = np.zeros(w.shape[0], dtype=np.float32)
-        #     print ("    bias shape:", np.shape(b))
         
         if 'BatchNorm2d' in str(m.type):
             b = m._parameters['bias'].data.numpy()
@@ -96,30 +108,24 @@ def print_wb_output(model, input_batch):
             rm = np.array(rm, dtype=np.float32)
             rv = m.running_var.data.numpy()
             rv = np.array(rv, dtype=np.float32)
-            #s.tofile(f, format="f")
             bin_write(f,b)
             bin_write(f,s)
             bin_write(f,rm)
             bin_write(f,rv)
+            print ("    b shape:", np.shape(b))
             print ("    s shape:", np.shape(s))
             print ("    rm shape:", np.shape(rm))
             print ("    rv shape:", np.shape(rv))
 
         else:
-            
-            # w.tofile(f, format="f")
             bin_write(f,w)
-            # print("w- ",w)
             if b.size > 0:
-                # b.tofile(f, format="f")
                 bin_write(f,b)
-                # print("b - ",b)
 
         if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type):
             f.close()
             print("close file")
             f = None
-            # return
 
 
 
@@ -130,34 +136,27 @@ if __name__ == '__main__':
     model = torch.hub.load('pytorch/vision', 'resnet101', pretrained=True)
     model.eval()
 
-    # Download an example image from the pytorch website
+    # load an example image and load it on model
-    url, filename = ("https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg")
+    model, input_batch = load_ex_image(model)
-    try: urllib.URLopener().retrieve(url, filename)
+    model.eval()
-    except: urllib.request.urlretrieve(url, filename)
-
-    # sample execution (requires torchvision)
-    input_image = Image.open(filename)
-    preprocess = transforms.Compose([
-        transforms.Resize(256),
-        transforms.CenterCrop(224),
-        transforms.ToTensor(),
-        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    input_tensor = preprocess(input_image)
-    input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model
-
-    # move the input and model to GPU for speed if available
-    if torch.cuda.is_available():
-        input_batch = input_batch.to('cuda')
-        model.to('cuda')
-
     with torch.no_grad():
         output = model(input_batch)
 
-    # Tensor of shape 1000, with confidence scores over Imagenet's 1000 classes
+    # create folders debug and layers if do not exist
-    # print(output)
+    create_folders()
 
+    # add output attribute to the layers
+    for n, m in model.named_modules():
+        m.register_forward_hook(hook)
 
-    print_wb_output(model, input_batch)
+    # export input bin
+    exp_input(model, input_batch)
 
-    # print(list(model.children()))
+    print_wb_output(model)
+
+    with open("resnet101.txt", 'w') as f:
+        for item in list(model.children()):
+            f.write("%s\n" % item)
+
+    summary(model, (3, 224, 224))
+    # print(trace(model, input_batch))