ELU implemented

src/Activation.cpp

@@ -1,10 +1,11 @@
 #include <iostream>
 
 #include "Layer.h"
+#include "kernels.h"
 
 namespace tkDNN {
 
-Activation::Activation(Network *net, dataDim_t input_dim, cudnnActivationMode_t act_mode) : 
+Activation::Activation(Network *net, dataDim_t input_dim, tkdnnActivationMode_t act_mode) : 
     Layer(net, input_dim) {
 
     this->act_mode = act_mode;
@@ -29,17 +30,21 @@ Activation::~Activation() {
 
 value_type* Activation::infer(dataDim_t &dim, value_type* srcData) {
 
-    value_type alpha = value_type(1);
-    value_type beta  = value_type(0);
-    checkCUDNN( cudnnActivationForward(net->cudnnHandle,
-                                        act_mode,
-                                        &alpha,
-                                        srcTensorDesc,
-                                        srcData,
-                                        &beta,
-                                        dstTensorDesc,
-                                        dstData) );    
+    if(act_mode == ACTIVATION_ELU) {
+        activationELUForward(srcData, dstData, dim.tot());
 
+    } else {
+        value_type alpha = value_type(1);
+        value_type beta  = value_type(0);
+        checkCUDNN( cudnnActivationForward(net->cudnnHandle,
+                                            cudnnActivationMode_t(act_mode),
+                                            &alpha,
+                                            srcTensorDesc,
+                                            srcData,
+                                            &beta,
+                                            dstTensorDesc,
+                                            dstData) );    
+    }
     return dstData;
 }
 

src/Dense.cpp

@@ -8,9 +8,14 @@ Dense::Dense(Network *net, dataDim_t in_dim,
     int out_ch, const char* fname_weights, const char* fname_bias) : 
     LayerWgs(net, in_dim, in_dim.tot(), out_ch, 1, 1, 1, fname_weights, fname_bias) {
 
-    this->out_ch = out_ch;
+    output_dim.n = 1;
+    output_dim.c = out_ch;
+    output_dim.h = 1;
+    output_dim.w = 1;
+    output_dim.l = 1;
+
     //allocate data for infer result
-    checkCuda( cudaMalloc(&dstData, outputs*sizeof(value_type)) );
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(value_type)) );
 }
 
 Dense::~Dense() {
@@ -24,9 +29,9 @@ value_type* Dense::infer(dataDim_t &dim, value_type* srcData) {
         FatalError("Not Implemented"); 
     
     int dim_x = dim.tot();
-    int dim_y = outputs;
+    int dim_y = output_dim.tot();
 
-    if (dim_x != inputs)
+    if (dim_x != input_dim.tot())
         FatalError("Input mismatch");
 
     value_type alpha = value_type(1), beta = value_type(1);

src/Layer.cpp

@@ -8,7 +8,8 @@ Layer::Layer(Network *net, dataDim_t in_dim) {
 
     this->net = net;
     this->input_dim = in_dim;
-
+    this->output_dim = in_dim;
+    
     checkCUDNN( cudnnCreateTensorDescriptor(&srcTensorDesc) );
     checkCUDNN( cudnnCreateTensorDescriptor(&dstTensorDesc) );
 }

src/kernels/activation_elu.cu

@@ -0,0 +1,17 @@
+#include "kernels.h"
+
+__global__
+void activation_elu(value_type *input, value_type *output, int size) {
+
+    int i = threadIdx.x*(blockIdx.x +1);
+    
+    if(i<size) 
+        output[i] =  (input[i]>0)*input[i] + (input[i]<0)*(expf(input[i]) -1);
+ }
+
+
+void activationELUForward(value_type* srcData, value_type* dstData, int size)
+{
+    activation_elu<<<(size+255)/256, 256>>>(srcData, dstData, size);
+    checkCuda( cudaDeviceSynchronize() );
+}