From d411af52fa9f6b9e853f14bb640724904195cd7a Mon Sep 17 00:00:00 2001
From: Harijs Grinbergs <1474290+TheExDeus@users.noreply.github.com>
Date: Mon, 1 Nov 2021 22:29:32 +0200
Subject: [PATCH] Small MISH optimization

- Small MISH optimization from darknet repo.
---
 src/kernels/activation_mish.cu | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/kernels/activation_mish.cu b/src/kernels/activation_mish.cu
index 75a39c9..661587d 100644
--- a/src/kernels/activation_mish.cu
+++ b/src/kernels/activation_mish.cu
@@ -35,6 +35,16 @@ float mish_yashas(float x) {
     return x - 2 * __fdividef(x, n + 2);
 }
 
+__device__ float mish_yashas2(float x)
+{
+    float e = __expf(x);
+    float n = e * e + 2 * e;
+    if (x <= -0.6f)
+        return x * __fdividef(n, n + 2);
+
+    return x - 2 * __fdividef(x, n + 2);
+}
+
 // https://github.com/digantamisra98/Mish
 // https://github.com/AlexeyAB/darknet/blob/master/src/activation_kernels.cu
 __global__
@@ -42,7 +52,7 @@ void activation_mish(dnnType *input, dnnType *output, int size) {
     int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
     if (i < size)
         // output[i] = input[i] * tanh_activate_kernel( softplus_kernel(input[i], MISH_THRESHOLD));
-        output[i] = mish_yashas(input[i]);
+        output[i] = mish_yashas2(input[i]);
 }
 
 /**